From 04d46de92d6c16f3b6ed35285aa36d61f2e5b1f4 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 17 Feb 2022 13:53:04 +0100 Subject: [PATCH 01/64] Add an empty application and a test suite --- src/kiss.app.src | 9 ++++++ src/kiss.erl | 77 +++++++++++++++++++++++++++++++++++++++++++++ src/kiss_proxy.erl | 31 ++++++++++++++++++ test/kiss_SUITE.erl | 25 +++++++++++++++ 4 files changed, 142 insertions(+) create mode 100644 src/kiss.app.src create mode 100644 src/kiss.erl create mode 100644 src/kiss_proxy.erl create mode 100644 test/kiss_SUITE.erl diff --git a/src/kiss.app.src b/src/kiss.app.src new file mode 100644 index 00000000..f0339234 --- /dev/null +++ b/src/kiss.app.src @@ -0,0 +1,9 @@ +{application, kiss, + [{description, "Simple Base"}, + {vsn, "0.1"}, + {modules, []}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []} +% {mod, {kiss_app, []} + ]}. diff --git a/src/kiss.erl b/src/kiss.erl new file mode 100644 index 00000000..a0aa0a37 --- /dev/null +++ b/src/kiss.erl @@ -0,0 +1,77 @@ +%% Very simple multinode ETS writer +%% One file, everything is simple, but we don't silently hide race conditions +%% No transactions +%% We don't use rpc module, because it is one gen_server + + +%% We don't use monitors to avoid round-trips (that's why we don't use calls neither) +-module(kiss). +-export([start/2]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-export([load_test/0]). + +-behaviour(gen_server). + +%% Table and server has the same name +start(Tab, Opts) when is_atom(Tab) -> + ets:new(Tab, [ordered_set, named_table, + public, {read_concurrency, true}]), + N = erlang:system_info(schedulers), + Ns = lists:seq(1, N), + Names = [list_to_atom(atom_to_list(Tab) ++ integer_to_list(NN)) || NN <- Ns], + persistent_term:put(Tab, list_to_tuple(Names)), + [gen_server:start({local, Name}, ?MODULE, [Tab], []) + || Name <- Names]. + + +stop(Tab) -> + gen_server:stop(Tab). + +load_test() -> + start(tab1, #{}), + timer:tc(fun() -> + pmap(fun(X) -> load_test2(1000000) end, lists:seq(1, erlang:system_info(schedulers))) + end). + +load_test2(0) -> ok; +load_test2(N) -> + S = erlang:system_info(scheduler_id), + Names = persistent_term:get(tab1), + Name = element(S, Names), + gen_server:call(Name, {insert, {N}}), + load_test2(N - 1). + +pmap(F, List) -> + Parent = self(), + Pids = [spawn_link(fun() -> Parent ! {result, self(), (catch F(X))} end) || X <- List], + [receive {result, P, Res} when Pid =:= P -> Res end + || Pid <- Pids]. + +%% Key is {USR, Sid, UpdateNumber} +%% Where first UpdateNumber is 0 +insert(Tab, Rec) -> +% Nodes = other_nodes(Tab), + ets:insert_new(Tab, Rec). + %% Insert to other nodes and block till written + +init([Tab]) -> + {ok, #{tab => Tab}}. + +handle_call({insert, Rec}, From, State = #{tab := Tab}) -> + ets:insert_new(Tab, Rec), + {reply, ok, State}. + +handle_cast(Msg, State) -> + {noreply, State}. + +handle_info(Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + diff --git a/src/kiss_proxy.erl b/src/kiss_proxy.erl new file mode 100644 index 00000000..9e6bbd1b --- /dev/null +++ b/src/kiss_proxy.erl @@ -0,0 +1,31 @@ +%% We monitor this process instead of a remote process +-module(kiss_proxy). +-export([start_link/1]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-behaviour(gen_server). + +start_link(RemotePid) -> + gen_server:start_link(?MODULE, [RemotePid], []). + +init([RemotePid]) -> + MonRef = erlang:monitor(process, RemotePid), + {ok, #{mon => MonRef, remote_pid => RemotePid}}. + +handle_call(_Reply, _From, State) -> + {reply, ok, State}. + +handle_cast(Msg, State) -> + {noreply, State}. + +handle_info({'DOWN', MonRef, process, Pid, _Reason}, State = #{mon := MonRef}) -> + error_logger:error_msg("Node down ~p", [node(Pid)]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl new file mode 100644 index 00000000..9eb77a60 --- /dev/null +++ b/test/kiss_SUITE.erl @@ -0,0 +1,25 @@ +-module(kiss_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile([export_all]). + +all() -> [ets_tests]. + +init_per_suite(Config) -> + {ok, Node2} = ct_slave:start(ct2, [{monitor_master, true}]), + [{node2, Node2}|Config]. + +end_per_suite(Config) -> + Config. + +init_per_testcase(ets_tests, Config) -> + Config. + +end_per_testcase(ets_tests, Config) -> + ok. + +ets_tests(Config) -> + Node2 = proplists:get_value(node2, Config), + kiss:start(tab1, #{}), + rpc:call(Node2, kiss, start, [tab1, #{}]), + ok. From eb2ee50bc94896e54e08ef0cd1ce5bd1ccae832e Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 17 Feb 2022 15:45:45 +0100 Subject: [PATCH 02/64] Add ETS replication logic (working simple test) Use persistent term for metadata Monitor a proxy process --- src/kiss.erl | 196 +++++++++++++++++++++++++++++++++++--------- src/kiss_proxy.erl | 16 ++-- src/kiss_pt.erl | 36 ++++++++ test/kiss_SUITE.erl | 66 +++++++++++++-- 4 files changed, 262 insertions(+), 52 deletions(-) create mode 100644 src/kiss_pt.erl diff --git a/src/kiss.erl b/src/kiss.erl index a0aa0a37..7081666e 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -2,76 +2,190 @@ %% One file, everything is simple, but we don't silently hide race conditions %% No transactions %% We don't use rpc module, because it is one gen_server +%% We monitor a proxy module (so, no remote monitors on each insert) +%% If we write in format {Key, WriterName}, we should resolve conflicts automatically. + %% We don't use monitors to avoid round-trips (that's why we don't use calls neither) -module(kiss). --export([start/2]). +-export([start/2, stop/1, dump/1, insert/2, join/2, other_nodes/1]). + -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --export([load_test/0]). - -behaviour(gen_server). %% Table and server has the same name -start(Tab, Opts) when is_atom(Tab) -> - ets:new(Tab, [ordered_set, named_table, - public, {read_concurrency, true}]), - N = erlang:system_info(schedulers), - Ns = lists:seq(1, N), - Names = [list_to_atom(atom_to_list(Tab) ++ integer_to_list(NN)) || NN <- Ns], - persistent_term:put(Tab, list_to_tuple(Names)), - [gen_server:start({local, Name}, ?MODULE, [Tab], []) - || Name <- Names]. - +start(Tab, _Opts) when is_atom(Tab) -> + gen_server:start({local, Tab}, ?MODULE, [Tab], []). stop(Tab) -> gen_server:stop(Tab). -load_test() -> - start(tab1, #{}), - timer:tc(fun() -> - pmap(fun(X) -> load_test2(1000000) end, lists:seq(1, erlang:system_info(schedulers))) - end). - -load_test2(0) -> ok; -load_test2(N) -> - S = erlang:system_info(scheduler_id), - Names = persistent_term:get(tab1), - Name = element(S, Names), - gen_server:call(Name, {insert, {N}}), - load_test2(N - 1). - -pmap(F, List) -> - Parent = self(), - Pids = [spawn_link(fun() -> Parent ! {result, self(), (catch F(X))} end) || X <- List], - [receive {result, P, Res} when Pid =:= P -> Res end - || Pid <- Pids]. +dump(Tab) -> + ets:tab2list(Tab). +join(RemotePid, Tab) when is_pid(RemotePid) -> + gen_server:call(Tab, {join, RemotePid}, infinity). + +remote_add_node_to_schema(RemotePid, ServerPid, OtherNodes) -> + gen_server:call(RemotePid, {remote_add_node_to_schema, ServerPid, OtherNodes}, infinity). + +remote_just_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> + gen_server:call(RemotePid, {remote_just_add_node_to_schema, ServerPid, OtherPids}, infinity). + +send_dump_to_remote_node(_RemotePid, _FromPid, []) -> + skipped; +send_dump_to_remote_node(RemotePid, FromPid, OurDump) -> + gen_server:call(RemotePid, {send_dump_to_remote_node, FromPid, OurDump}, infinity). + +%% Inserts do not override data (i.e. immunable) +%% But we can remove data %% Key is {USR, Sid, UpdateNumber} %% Where first UpdateNumber is 0 insert(Tab, Rec) -> -% Nodes = other_nodes(Tab), - ets:insert_new(Tab, Rec). + Servers = other_servers(Tab), + ets:insert(Tab, Rec), %% Insert to other nodes and block till written + Monitors = insert_to_remote_nodes(Servers, Rec), + wait_for_inserted(Monitors). + +insert_to_remote_nodes([{RemotePid, ProxyPid} | Servers], Rec) -> + Mon = erlang:monitor(process, ProxyPid), + erlang:send(RemotePid, {insert_from_remote_node, Mon, self(), Rec}, [noconnect]), + [Mon | insert_to_remote_nodes(Servers, Rec)]; +insert_to_remote_nodes([], _Rec) -> + []. + +wait_for_inserted([Mon | Monitors]) -> + receive + {inserted, Mon2} when Mon2 =:= Mon -> + wait_for_inserted(Monitors); + {'DOWN', Mon2, process, _Pid, _Reason} when Mon2 =:= Mon -> + wait_for_inserted(Monitors) + end; +wait_for_inserted([]) -> + ok. -init([Tab]) -> - {ok, #{tab => Tab}}. +other_servers(Tab) -> +% gen_server:call(Tab, get_other_servers). + kiss_pt:get(Tab). + +other_nodes(Tab) -> + lists:sort([node(Pid) || {Pid, _} <- other_servers(Tab)]). -handle_call({insert, Rec}, From, State = #{tab := Tab}) -> - ets:insert_new(Tab, Rec), +init([Tab]) -> + ets:new(Tab, [ordered_set, named_table, + public, {read_concurrency, true}]), + update_pt(Tab, []), + {ok, #{tab => Tab, other_servers => []}}. + +handle_call({join, RemotePid}, _From, State) -> + handle_join(RemotePid, State); +handle_call({remote_add_node_to_schema, ServerPid, OtherPids}, _From, State) -> + handle_remote_add_node_to_schema(ServerPid, OtherPids, State); +handle_call({remote_just_add_node_to_schema, ServerPid, OtherPids}, _From, State) -> + handle_remote_just_add_node_to_schema(ServerPid, OtherPids, State); +handle_call({send_dump_to_remote_node, FromPid, Dump}, _From, State) -> + handle_send_dump_to_remote_node(FromPid, Dump, State); +handle_call(get_other_servers, _From, State = #{other_servers := Servers}) -> + {reply, Servers, State}; +handle_call({insert, Rec}, _From, State = #{tab := Tab}) -> + ets:insert(Tab, Rec), {reply, ok, State}. -handle_cast(Msg, State) -> +handle_cast(_Msg, State) -> {noreply, State}. -handle_info(Info, State) -> +handle_info({'DOWN', _Mon, Pid, _Reason}, State) -> + handle_down(Pid, State); +handle_info({insert_from_remote_node, Mon, Pid, Rec}, State = #{tab := Tab}) -> + ets:insert(Tab, Rec), + Pid ! {inserted, Mon}, {noreply, State}. -terminate(_Reason, _State) -> +terminate(_Reason, _State = #{tab := Tab}) -> + kiss_pt:put(Tab, []), ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. + +handle_join(RemotePid, State = #{tab := Tab, other_servers := Servers}) when is_pid(RemotePid) -> + case lists:keymember(RemotePid, 1, Servers) of + true -> + %% Already added + {reply, ok, State}; + false -> + KnownPids = [Pid || {Pid, _} <- Servers], + %% TODO can crash + case remote_add_node_to_schema(RemotePid, self(), KnownPids) of + {ok, Dump, OtherPids} -> + %% Let all nodes to know each other + [remote_just_add_node_to_schema(Pid, self(), KnownPids) || Pid <- OtherPids], + [remote_just_add_node_to_schema(Pid, self(), [RemotePid | OtherPids]) || Pid <- KnownPids], + Servers2 = lists:usort(start_proxies_for([RemotePid | OtherPids], Servers) ++ Servers), + %% Ask our node to replicate data there before applying the dump + update_pt(Tab, Servers2), + OurDump = dump(Tab), + %% Send to all nodes from that partition + [send_dump_to_remote_node(Pid, self(), OurDump) || Pid <- [RemotePid | OtherPids]], + %% Apply to our nodes + [send_dump_to_remote_node(Pid, self(), Dump) || Pid <- KnownPids], + insert_many(Tab, Dump), + %% Add ourself into remote schema + %% Add remote nodes into our schema + %% Copy from our node / Copy into our node + {reply, ok, State#{other_servers => Servers2}}; + Other -> + error_logger:error_msg("remote_add_node_to_schema failed ~p", [Other]), + {reply, {error, remote_add_node_to_schema_failed}, State} + end + end. + +handle_remote_add_node_to_schema(ServerPid, OtherPids, State = #{tab := Tab}) -> + case handle_remote_just_add_node_to_schema(ServerPid, OtherPids, State) of + {reply, {ok, KnownPids}, State2} -> + {reply, {ok, dump(Tab), KnownPids}, State2}; + Other -> + Other + end. + +handle_remote_just_add_node_to_schema(RemotePid, OtherPids, State = #{tab := Tab, other_servers := Servers}) -> + Servers2 = lists:usort(start_proxies_for([RemotePid | OtherPids], Servers) ++ Servers), + update_pt(Tab, Servers2), + KnownPids = [Pid || {Pid, _} <- Servers], + {reply, {ok, KnownPids}, State#{other_servers => Servers2}}. + +start_proxies_for([RemotePid | OtherPids], AlreadyAddedNodes) + when is_pid(RemotePid), RemotePid =/= self() -> + case lists:keymember(RemotePid, 1, AlreadyAddedNodes) of + false -> + {ok, ProxyPid} = kiss_proxy:start(RemotePid), + erlang:monitor(process, ProxyPid), + [{RemotePid, ProxyPid} | start_proxies_for(OtherPids, AlreadyAddedNodes)]; + true -> + error_logger:info_msg("what=already_added remote_pid=~p node=~p", [RemotePid, node(RemotePid)]), + start_proxies_for(OtherPids, AlreadyAddedNodes) + end; +start_proxies_for([], _AlreadyAddedNodes) -> + []. + +handle_send_dump_to_remote_node(_FromPid, Dump, State = #{tab := Tab}) -> + insert_many(Tab, Dump), + {reply, ok, State}. + +insert_many(Tab, Recs) -> + ets:insert(Tab, Recs). + +handle_down(Pid, State = #{tab := Tab, other_servers := Servers}) -> + %% Down from a proxy + Servers2 = lists:keydelete(Pid, 2, Servers), + update_pt(Tab, Servers2), + {noreply, State#{other_servers => Servers2}}. + +%% Called each time other_servers changes +update_pt(Tab, Servers2) -> + kiss_pt:put(Tab, Servers2). diff --git a/src/kiss_proxy.erl b/src/kiss_proxy.erl index 9e6bbd1b..d76dee4c 100644 --- a/src/kiss_proxy.erl +++ b/src/kiss_proxy.erl @@ -1,17 +1,18 @@ %% We monitor this process instead of a remote process -module(kiss_proxy). --export([start_link/1]). +-export([start/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -behaviour(gen_server). -start_link(RemotePid) -> - gen_server:start_link(?MODULE, [RemotePid], []). +start(RemotePid) -> + gen_server:start(?MODULE, [RemotePid, self()], []). -init([RemotePid]) -> +init([RemotePid, ParentPid]) -> MonRef = erlang:monitor(process, RemotePid), - {ok, #{mon => MonRef, remote_pid => RemotePid}}. + MonRef2 = erlang:monitor(process, ParentPid), + {ok, #{mon => MonRef, pmon => MonRef2, remote_pid => RemotePid}}. handle_call(_Reply, _From, State) -> {reply, ok, State}. @@ -21,7 +22,10 @@ handle_cast(Msg, State) -> handle_info({'DOWN', MonRef, process, Pid, _Reason}, State = #{mon := MonRef}) -> error_logger:error_msg("Node down ~p", [node(Pid)]), - {noreply, State}. + {stop, State}; +handle_info({'DOWN', MonRef, process, Pid, _Reason}, State = #{pmon := MonRef}) -> + error_logger:error_msg("Parent down ~p", [Pid]), + {stop, State}. terminate(_Reason, _State) -> ok. diff --git a/src/kiss_pt.erl b/src/kiss_pt.erl new file mode 100644 index 00000000..34af5cb1 --- /dev/null +++ b/src/kiss_pt.erl @@ -0,0 +1,36 @@ +-module(kiss_pt). +-export([put/2, get/1]). + +-compile({no_auto_import,[get/1]}). + +%% Avoids GC +put(Tab, Data) -> + try get(Tab) of + Data -> + ok; + _ -> + just_put(Tab, Data) + catch _:_ -> + just_put(Tab, Data) + end. + +just_put(Tab, Data) -> + NextKey = next_key(Tab), + persistent_term:put(NextKey, Data), + persistent_term:put(Tab, NextKey). + +get(Tab) -> + Key = persistent_term:get(Tab), + persistent_term:get(Key). + +next_key(Tab) -> + try + Key = persistent_term:get(Tab), + N = list_to_integer(get_suffix(atom_to_list(Key), atom_to_list(Tab) ++ "_")) + 1, + list_to_atom(atom_to_list(Tab) ++ "_" ++ integer_to_list(N)) + catch _:_ -> + list_to_atom(atom_to_list(Tab) ++ "_1") + end. + +get_suffix(Str, Prefix) -> + lists:sublist(Str, length(Prefix) + 1, length(Str) - length(Prefix)). diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index 9eb77a60..51aa2b9d 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -6,8 +6,10 @@ all() -> [ets_tests]. init_per_suite(Config) -> - {ok, Node2} = ct_slave:start(ct2, [{monitor_master, true}]), - [{node2, Node2}|Config]. + Node2 = start_node(ct2), + Node3 = start_node(ct3), + Node4 = start_node(ct4), + [{nodes, [Node2, Node3, Node4]}|Config]. end_per_suite(Config) -> Config. @@ -19,7 +21,61 @@ end_per_testcase(ets_tests, Config) -> ok. ets_tests(Config) -> - Node2 = proplists:get_value(node2, Config), - kiss:start(tab1, #{}), - rpc:call(Node2, kiss, start, [tab1, #{}]), + Node1 = node(), + [Node2, Node3, Node4] = proplists:get_value(nodes, Config), + Tab = tab1, + {ok, Pid1} = start(Node1, Tab), + {ok, Pid2} = start(Node2, Tab), + {ok, Pid3} = start(Node3, Tab), + {ok, Pid4} = start(Node4, Tab), + join(Node1, Pid3, Tab), + join(Node2, Pid4, Tab), + insert(Node1, Tab, {a}), + insert(Node2, Tab, {b}), + insert(Node3, Tab, {c}), + insert(Node4, Tab, {d}), + [{a},{c}] = dump(Node1, Tab), + [{b},{d}] = dump(Node2, Tab), + join(Node1, Pid2, Tab), + [{a},{b},{c},{d}] = dump(Node1, Tab), + [{a},{b},{c},{d}] = dump(Node2, Tab), + insert(Node1, Tab, {f}), + insert(Node4, Tab, {e}), + AF = [{a},{b},{c},{d},{e},{f}], + AF = dump(Node1, Tab), + AF = dump(Node2, Tab), + AF = dump(Node3, Tab), + AF = dump(Node4, Tab), + [Node2, Node3, Node4] = other_nodes(Node1, Tab), + [Node1, Node3, Node4] = other_nodes(Node2, Tab), + [Node1, Node2, Node4] = other_nodes(Node3, Tab), + [Node1, Node2, Node3] = other_nodes(Node4, Tab), ok. + +start(Node, Tab) -> + rpc(Node, kiss, start, [Tab, #{}]). + +insert(Node, Tab, Rec) -> + rpc(Node, kiss, insert, [Tab, Rec]). + +dump(Node, Tab) -> + rpc(Node, kiss, dump, [Tab]). + +other_nodes(Node, Tab) -> + rpc(Node, kiss, other_nodes, [Tab]). + +join(Node1, Node2, Tab) -> + rpc(Node1, kiss, join, [Node2, Tab]). + +rpc(Node, M, F, Args) -> + case rpc:call(Node, M, F, Args) of + {badrpc, Error} -> + ct:fail({badrpc, Error}); + Other -> + Other + end. + +start_node(Sname) -> + {ok, Node} = ct_slave:start(Sname, [{monitor_master, true}]), + rpc:call(Node, code, add_paths, [code:get_path()]), + Node. From b5eedc8746326ae92fde3242b007959d01265a9e Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 21 Feb 2022 13:19:32 +0100 Subject: [PATCH 03/64] Add logging and long task tracking --- README.md | 4 ++++ rebar.lock | 1 + run_test.sh | 2 ++ src/kiss.erl | 27 +++++++++++++++++++++------ src/kiss_long.erl | 40 ++++++++++++++++++++++++++++++++++++++++ test/kiss_SUITE.erl | 20 +++++++++++++++----- 6 files changed, 83 insertions(+), 11 deletions(-) create mode 100644 README.md create mode 100644 rebar.lock create mode 100755 run_test.sh create mode 100644 src/kiss_long.erl diff --git a/README.md b/README.md new file mode 100644 index 00000000..07b7f0d2 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +``` +./rebar3 ct --sname=ct +``` + diff --git a/rebar.lock b/rebar.lock new file mode 100644 index 00000000..57afcca0 --- /dev/null +++ b/rebar.lock @@ -0,0 +1 @@ +[]. diff --git a/run_test.sh b/run_test.sh new file mode 100755 index 00000000..9a214163 --- /dev/null +++ b/run_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +./rebar3 ct --sname=ct1 diff --git a/src/kiss.erl b/src/kiss.erl index 7081666e..e8436cc0 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -6,6 +6,8 @@ %% If we write in format {Key, WriterName}, we should resolve conflicts automatically. +%% +%% While Tab is an atom, we can join tables with different atoms for the local testing. %% We don't use monitors to avoid round-trips (that's why we don't use calls neither) -module(kiss). @@ -26,19 +28,29 @@ stop(Tab) -> dump(Tab) -> ets:tab2list(Tab). -join(RemotePid, Tab) when is_pid(RemotePid) -> - gen_server:call(Tab, {join, RemotePid}, infinity). +%% Adds a node to a cluster. +%% Writes from other nodes would wait for join completion. +join(Tab, RemotePid) when is_pid(RemotePid) -> + F = fun() -> gen_server:call(Tab, {join, RemotePid}, infinity) end, + kiss_long:run("task=join table=~p remote_pid=~p remote_node=~p ", + [Tab, RemotePid, node(RemotePid)], F). -remote_add_node_to_schema(RemotePid, ServerPid, OtherNodes) -> - gen_server:call(RemotePid, {remote_add_node_to_schema, ServerPid, OtherNodes}, infinity). +remote_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> + F = fun() -> gen_server:call(RemotePid, {remote_add_node_to_schema, ServerPid, OtherPids}, infinity) end, + kiss_long:run("task=remote_add_node_to_schema remote_pid=~p remote_node=~p other_pids=~0p other_nodes=~0p ", + [RemotePid, node(RemotePid), OtherPids, pids_to_nodes(OtherPids)], F). remote_just_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> - gen_server:call(RemotePid, {remote_just_add_node_to_schema, ServerPid, OtherPids}, infinity). + F = fun() -> gen_server:call(RemotePid, {remote_just_add_node_to_schema, ServerPid, OtherPids}, infinity) end, + kiss_long:run("task=remote_just_add_node_to_schema remote_pid=~p remote_node=~p other_pids=~0p other_nodes=~0p ", + [RemotePid, node(RemotePid), OtherPids, pids_to_nodes(OtherPids)], F). send_dump_to_remote_node(_RemotePid, _FromPid, []) -> skipped; send_dump_to_remote_node(RemotePid, FromPid, OurDump) -> - gen_server:call(RemotePid, {send_dump_to_remote_node, FromPid, OurDump}, infinity). + F = fun() -> gen_server:call(RemotePid, {send_dump_to_remote_node, FromPid, OurDump}, infinity) end, + kiss_long:run("task=send_dump_to_remote_node remote_pid=~p count=~p ", + [RemotePid, length(OurDump)], F). %% Inserts do not override data (i.e. immunable) %% But we can remove data @@ -189,3 +201,6 @@ handle_down(Pid, State = #{tab := Tab, other_servers := Servers}) -> %% Called each time other_servers changes update_pt(Tab, Servers2) -> kiss_pt:put(Tab, Servers2). + +pids_to_nodes(Pids) -> + lists:map(fun node/1, Pids). diff --git a/src/kiss_long.erl b/src/kiss_long.erl new file mode 100644 index 00000000..fa0ac1c8 --- /dev/null +++ b/src/kiss_long.erl @@ -0,0 +1,40 @@ +-module(kiss_long). +-export([run/3]). + +run(InfoText, InfoArgs, Fun) -> + Parent = self(), + Start = os:timestamp(), + error_logger:info_msg("what=long_task_started " ++ InfoText, InfoArgs), + Pid = spawn_mon(InfoText, InfoArgs, Parent, Start), + try + Fun() + after + Diff = diff(Start), + error_logger:info_msg("what=long_task_finished time=~p ms " ++ InfoText, + [Diff] ++ InfoArgs), + Pid ! stop + end. + +spawn_mon(InfoText, InfoArgs, Parent, Start) -> + spawn_link(fun() -> run_monitor(InfoText, InfoArgs, Parent, Start) end). + +run_monitor(InfoText, InfoArgs, Parent, Start) -> + Mon = erlang:monitor(process, Parent), + monitor_loop(Mon, InfoText, InfoArgs, Start). + +monitor_loop(Mon, InfoText, InfoArgs, Start) -> + receive + {'DOWN', MonRef, process, _Pid, Reason} when Mon =:= MonRef -> + error_logger:error_msg("what=long_task_failed reason=~p " ++ InfoText, + [Reason] ++ InfoArgs), + ok; + stop -> ok + after 5000 -> + Diff = diff(Start), + error_logger:info_msg("what=long_task_progress time=~p ms " ++ InfoText, + [Diff] ++ InfoArgs), + monitor_loop(Mon, InfoText, InfoArgs, Start) + end. + +diff(Start) -> + timer:now_diff(os:timestamp(), Start) div 1000. diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index 51aa2b9d..f22ab20b 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -3,7 +3,7 @@ -compile([export_all]). -all() -> [ets_tests]. +all() -> [test_multinode, test_locally]. init_per_suite(Config) -> Node2 = start_node(ct2), @@ -14,13 +14,13 @@ init_per_suite(Config) -> end_per_suite(Config) -> Config. -init_per_testcase(ets_tests, Config) -> +init_per_testcase(_, Config) -> Config. -end_per_testcase(ets_tests, Config) -> +end_per_testcase(_, Config) -> ok. -ets_tests(Config) -> +test_multinode(Config) -> Node1 = node(), [Node2, Node3, Node4] = proplists:get_value(nodes, Config), Tab = tab1, @@ -52,6 +52,16 @@ ets_tests(Config) -> [Node1, Node2, Node3] = other_nodes(Node4, Tab), ok. +test_locally(Config) -> + {ok, Pid1} = kiss:start(t1, #{}), + {ok, Pid2} = kiss:start(t2, #{}), + kiss:join(t1, Pid2), + kiss:insert(t1, {1}), + kiss:insert(t1, {1}), + kiss:insert(t2, {2}), + D = kiss:dump(t1), + D = kiss:dump(t2). + start(Node, Tab) -> rpc(Node, kiss, start, [Tab, #{}]). @@ -65,7 +75,7 @@ other_nodes(Node, Tab) -> rpc(Node, kiss, other_nodes, [Tab]). join(Node1, Node2, Tab) -> - rpc(Node1, kiss, join, [Node2, Tab]). + rpc(Node1, kiss, join, [Tab, Node2]). rpc(Node, M, F, Args) -> case rpc:call(Node, M, F, Args) of From 5951b7db36a84aeff4e3c3a8e4a48d48b0feb521 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 21 Feb 2022 13:31:22 +0100 Subject: [PATCH 04/64] Lock joining Report time to get the joining lock --- src/kiss.erl | 28 +++++++++++++++++++++++++--- test/kiss_SUITE.erl | 12 ++++++------ 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index e8436cc0..4395ed65 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -11,7 +11,7 @@ %% We don't use monitors to avoid round-trips (that's why we don't use calls neither) -module(kiss). --export([start/2, stop/1, dump/1, insert/2, join/2, other_nodes/1]). +-export([start/2, stop/1, dump/1, insert/2, join/3, other_nodes/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). @@ -30,11 +30,33 @@ dump(Tab) -> %% Adds a node to a cluster. %% Writes from other nodes would wait for join completion. -join(Tab, RemotePid) when is_pid(RemotePid) -> - F = fun() -> gen_server:call(Tab, {join, RemotePid}, infinity) end, +%% LockKey should be the same on all nodes. +join(LockKey, Tab, RemotePid) when is_pid(RemotePid) -> + Start = os:timestamp(), + F = fun() -> join_loop(LockKey, Tab, RemotePid, Start) end, kiss_long:run("task=join table=~p remote_pid=~p remote_node=~p ", [Tab, RemotePid, node(RemotePid)], F). +join_loop(LockKey, Tab, RemotePid, Start) -> + F = fun() -> + Diff = timer:now_diff(os:timestamp(), Start) div 1000, + %% Getting the lock could take really long time in case nodes are + %% overloaded or joining is already in progress on another node + error_logger:info_msg("what=join_got_lock table=~p after_time=~p ms", [Tab, Diff]), + gen_server:call(Tab, {join, RemotePid}, infinity) + end, + LockRequest = {LockKey, self()}, + %% Just lock all nodes, no magic here :) + Nodes = [node() | nodes()], + Retries = 1, + case global:trans(LockRequest, F, Nodes, Retries) of + aborted -> + error_logger:error_msg("what=join_retry reason=lock_aborted", []), + join_loop(LockKey, Tab, RemotePid, Start); + Result -> + Result + end. + remote_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> F = fun() -> gen_server:call(RemotePid, {remote_add_node_to_schema, ServerPid, OtherPids}, infinity) end, kiss_long:run("task=remote_add_node_to_schema remote_pid=~p remote_node=~p other_pids=~0p other_nodes=~0p ", diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index f22ab20b..5b1f08b7 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -17,14 +17,14 @@ end_per_suite(Config) -> init_per_testcase(_, Config) -> Config. -end_per_testcase(_, Config) -> +end_per_testcase(_, _Config) -> ok. test_multinode(Config) -> Node1 = node(), [Node2, Node3, Node4] = proplists:get_value(nodes, Config), Tab = tab1, - {ok, Pid1} = start(Node1, Tab), + {ok, _Pid1} = start(Node1, Tab), {ok, Pid2} = start(Node2, Tab), {ok, Pid3} = start(Node3, Tab), {ok, Pid4} = start(Node4, Tab), @@ -52,10 +52,10 @@ test_multinode(Config) -> [Node1, Node2, Node3] = other_nodes(Node4, Tab), ok. -test_locally(Config) -> - {ok, Pid1} = kiss:start(t1, #{}), +test_locally(_Config) -> + {ok, _Pid1} = kiss:start(t1, #{}), {ok, Pid2} = kiss:start(t2, #{}), - kiss:join(t1, Pid2), + kiss:join(lock1, t1, Pid2), kiss:insert(t1, {1}), kiss:insert(t1, {1}), kiss:insert(t2, {2}), @@ -75,7 +75,7 @@ other_nodes(Node, Tab) -> rpc(Node, kiss, other_nodes, [Tab]). join(Node1, Node2, Tab) -> - rpc(Node1, kiss, join, [Tab, Node2]). + rpc(Node1, kiss, join, [lock1, Tab, Node2]). rpc(Node, M, F, Args) -> case rpc:call(Node, M, F, Args) of From e3011b4f0544dd5d9c7cdf0cf45eb92ec269b538 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 21 Feb 2022 15:52:01 +0100 Subject: [PATCH 05/64] Add auto-discovery support --- src/kiss.erl | 14 +++++--- src/kiss_discovery.erl | 74 ++++++++++++++++++++++++++++++++++++++++++ test/kiss_SUITE.erl | 21 +++++++++++- 3 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 src/kiss_discovery.erl diff --git a/src/kiss.erl b/src/kiss.erl index 4395ed65..408b6cb9 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -32,10 +32,16 @@ dump(Tab) -> %% Writes from other nodes would wait for join completion. %% LockKey should be the same on all nodes. join(LockKey, Tab, RemotePid) when is_pid(RemotePid) -> - Start = os:timestamp(), - F = fun() -> join_loop(LockKey, Tab, RemotePid, Start) end, - kiss_long:run("task=join table=~p remote_pid=~p remote_node=~p ", - [Tab, RemotePid, node(RemotePid)], F). + Servers = other_servers(Tab), + case lists:keymember(RemotePid, 1, Servers) of + true -> + {error, already_joined}; + false -> + Start = os:timestamp(), + F = fun() -> join_loop(LockKey, Tab, RemotePid, Start) end, + kiss_long:run("task=join table=~p remote_pid=~p remote_node=~p ", + [Tab, RemotePid, node(RemotePid)], F) + end. join_loop(LockKey, Tab, RemotePid, Start) -> F = fun() -> diff --git a/src/kiss_discovery.erl b/src/kiss_discovery.erl new file mode 100644 index 00000000..62b6ffd6 --- /dev/null +++ b/src/kiss_discovery.erl @@ -0,0 +1,74 @@ +%% AWS autodiscovery is kinda bad. +%% - UDP broadcasts do not work +%% - AWS CLI needs access +%% - DNS does not allow to list subdomains +%% So, we use a file with nodes to connect as a discovery mechanism +%% (so, you can hardcode nodes or use your method of filling it) +-module(kiss_discovery). +-export([start/1]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-behaviour(gen_server). + +%% disco_file +%% tables +start(Opts = #{disco_file := _, tables := _}) -> + gen_server:start(?MODULE, [Opts], []). + +init([Opts]) -> + self() ! check, + {ok, Opts#{results => []}}. + +handle_call(_Reply, _From, State) -> + {reply, ok, State}. + +handle_cast(Msg, State) -> + {noreply, State}. + +handle_info(check, State) -> + {noreply, handle_check(State)}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +handle_check(State = #{disco_file := Filename, tables := Tables}) -> + State2 = case file:read_file(Filename) of + {error, Reason} -> + error_logger:error_msg("what=discovery_failed filename=~0p reason=~0p", + [Filename, Reason]), + State; + {ok, Text} -> + Lines = binary:split(Text, [<<"\r">>, <<"\n">>, <<" ">>], [global]), + Nodes = [binary_to_atom(X, latin1) || X <- Lines, X =/= <<>>], + Results = [do_join(Tab, Node) || Tab <- Tables, Node <- Nodes, node() =/= Node], + report_results(Results, State), + State#{results => Results} + end, + schedule_check(), + State2. + +schedule_check() -> + erlang:send_after(5000, self(), check). + + +do_join(Tab, Node) -> + %% That would trigger autoconnect for the first time + case rpc:call(Node, erlang, whereis, [Tab]) of + Pid when is_pid(Pid) -> + Result = kiss:join(kiss_discovery, Tab, Pid), + #{what => join_result, result => Result, node => Node, table => Tab}; + Other -> + #{what => pid_not_found, reason => Other, node => Node, table => Tab} + end. + +report_results(Results, State = #{results := OldResults}) -> + Changed = Results -- OldResults, + [report_result(Result) || Result <- Results]. + +report_result(Map) -> + Text = [io_lib:format("~0p=~0p ", [K, V]) || {K, V} <- maps:to_list(Map)], + error_logger:info_msg("discovery ~s", [Text]). diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index 5b1f08b7..3c41e5c8 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -3,7 +3,7 @@ -compile([export_all]). -all() -> [test_multinode, test_locally]. +all() -> [test_multinode, test_multinode_auto_discovery, test_locally]. init_per_suite(Config) -> Node2 = start_node(ct2), @@ -14,6 +14,9 @@ init_per_suite(Config) -> end_per_suite(Config) -> Config. +init_per_testcase(test_multinode_auto_discovery, Config) -> + ct:make_priv_dir(), + Config; init_per_testcase(_, Config) -> Config. @@ -52,6 +55,22 @@ test_multinode(Config) -> [Node1, Node2, Node3] = other_nodes(Node4, Tab), ok. +test_multinode_auto_discovery(Config) -> + Node1 = node(), + [Node2, Node3, Node4] = proplists:get_value(nodes, Config), + Tab = tab2, + {ok, _Pid1} = start(Node1, Tab), + {ok, Pid2} = start(Node2, Tab), + Dir = proplists:get_value(priv_dir, Config), + ct:pal("Dir ~p", [Dir]), + FileName = filename:join(Dir, "disco.txt"), + ok = file:write_file(FileName, io_lib:format("~s~n~s~n", [Node1, Node2])), + {ok, Disco} = kiss_discovery:start(#{tables => [Tab], disco_file => FileName}), + %% Waits for the first check + ok = gen_server:call(Disco, ping), + [Node2] = other_nodes(Node1, Tab), + ok. + test_locally(_Config) -> {ok, _Pid1} = kiss:start(t1, #{}), {ok, Pid2} = kiss:start(t2, #{}), From 36c75e45f177d7d80457cca1a2a0c76aaf238344 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 21 Feb 2022 16:41:56 +0100 Subject: [PATCH 06/64] Add handle_down --- src/kiss.erl | 57 ++++++++++++++++++++++++++++++++++----------- test/kiss_SUITE.erl | 17 +++++++++++++- 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 408b6cb9..55a487e6 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -19,8 +19,15 @@ -behaviour(gen_server). %% Table and server has the same name -start(Tab, _Opts) when is_atom(Tab) -> - gen_server:start({local, Tab}, ?MODULE, [Tab], []). +%% Opts: +%% - handle_down = fun(#{remote_pid => Pid, table => Tab}) +%% Called when a remote node goes down. Do not update other nodes data +%% from this function (otherwise circular locking could happen - use spawn +%% to make a new async process if you need to update). +%% i.e. any functions that replicate changes are not allowed (i.e. insert/2, +%% remove/2). +start(Tab, Opts) when is_atom(Tab) -> + gen_server:start({local, Tab}, ?MODULE, [Tab, Opts], []). stop(Tab) -> gen_server:stop(Tab). @@ -80,10 +87,8 @@ send_dump_to_remote_node(RemotePid, FromPid, OurDump) -> kiss_long:run("task=send_dump_to_remote_node remote_pid=~p count=~p ", [RemotePid, length(OurDump)], F). -%% Inserts do not override data (i.e. immunable) -%% But we can remove data -%% Key is {USR, Sid, UpdateNumber} -%% Where first UpdateNumber is 0 +%% Only the node that owns the data could update/remove the data. +%% Key is {USR, Sid} insert(Tab, Rec) -> Servers = other_servers(Tab), ets:insert(Tab, Rec), @@ -115,11 +120,11 @@ other_servers(Tab) -> other_nodes(Tab) -> lists:sort([node(Pid) || {Pid, _} <- other_servers(Tab)]). -init([Tab]) -> +init([Tab, Opts]) -> ets:new(Tab, [ordered_set, named_table, public, {read_concurrency, true}]), update_pt(Tab, []), - {ok, #{tab => Tab, other_servers => []}}. + {ok, #{tab => Tab, other_servers => [], opts => Opts}}. handle_call({join, RemotePid}, _From, State) -> handle_join(RemotePid, State); @@ -138,7 +143,7 @@ handle_call({insert, Rec}, _From, State = #{tab := Tab}) -> handle_cast(_Msg, State) -> {noreply, State}. -handle_info({'DOWN', _Mon, Pid, _Reason}, State) -> +handle_info({'DOWN', _Mon, process, Pid, _Reason}, State) -> handle_down(Pid, State); handle_info({insert_from_remote_node, Mon, Pid, Rec}, State = #{tab := Tab}) -> ets:insert(Tab, Rec), @@ -220,11 +225,18 @@ handle_send_dump_to_remote_node(_FromPid, Dump, State = #{tab := Tab}) -> insert_many(Tab, Recs) -> ets:insert(Tab, Recs). -handle_down(Pid, State = #{tab := Tab, other_servers := Servers}) -> - %% Down from a proxy - Servers2 = lists:keydelete(Pid, 2, Servers), - update_pt(Tab, Servers2), - {noreply, State#{other_servers => Servers2}}. +handle_down(ProxyPid, State = #{tab := Tab, other_servers := Servers}) -> + case lists:keytake(ProxyPid, 2, Servers) of + {value, {RemotePid, _}, Servers2} -> + %% Down from a proxy + update_pt(Tab, Servers2), + call_user_handle_down(RemotePid, State), + {noreply, State#{other_servers => Servers2}}; + false -> + %% This should not happen + error_logger:error_msg("handle_down failed proxy_pid=~p state=~0p", [ProxyPid, State]), + {noreply, State} + end. %% Called each time other_servers changes update_pt(Tab, Servers2) -> @@ -232,3 +244,20 @@ update_pt(Tab, Servers2) -> pids_to_nodes(Pids) -> lists:map(fun node/1, Pids). + +%% Cleanup +call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> + case Opts of + #{handle_down := F} -> + try + FF = fun() -> F(#{remote_pid => RemotePid, table => Tab}) end, + kiss_long:run("task=call_user_handle_down table=~p remote_pid=~p remote_node=~p ", + [Tab, RemotePid, node(RemotePid)], FF) + catch Class:Error:Stacktrace -> + error_logger:error_msg("what=call_user_handle_down_failed table=~p " + "remote_pid=~p remote_node=~p class=~p reason=~0p stacktrace=~0p", + [Tab, RemotePid, node(RemotePid), Class, Error, Stacktrace]) + end; + _ -> + ok + end. diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index 3c41e5c8..80e7aa9e 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -3,7 +3,8 @@ -compile([export_all]). -all() -> [test_multinode, test_multinode_auto_discovery, test_locally]. +all() -> [test_multinode, test_multinode_auto_discovery, test_locally, + handle_down_is_called]. init_per_suite(Config) -> Node2 = start_node(ct2), @@ -81,6 +82,20 @@ test_locally(_Config) -> D = kiss:dump(t1), D = kiss:dump(t2). +handle_down_is_called(_Config) -> + Parent = self(), + DownFn = fun(#{remote_pid := _RemotePid, table := _Tab}) -> + Parent ! down_called + end, + {ok, Pid1} = kiss:start(d1, #{handle_down => DownFn}), + {ok, Pid2} = kiss:start(d2, #{}), + kiss:join(lock1, d1, Pid2), + exit(Pid2, oops), + receive + down_called -> ok + after 5000 -> ct:fail(timeout) + end. + start(Node, Tab) -> rpc(Node, kiss, start, [Tab, #{}]). From 89d4caa9f520559810d5c7d2253451e628465c0d Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 21 Feb 2022 18:38:59 +0100 Subject: [PATCH 07/64] Add delete and multidelete --- src/kiss.erl | 45 ++++++++++++++++++++++++++++++++++++--------- test/kiss_SUITE.erl | 35 ++++++++++++++++++++++++++--------- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 55a487e6..5389fda0 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -11,7 +11,7 @@ %% We don't use monitors to avoid round-trips (that's why we don't use calls neither) -module(kiss). --export([start/2, stop/1, dump/1, insert/2, join/3, other_nodes/1]). +-export([start/2, stop/1, dump/1, insert/2, delete/2, delete_many/2, join/3, other_nodes/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). @@ -88,13 +88,13 @@ send_dump_to_remote_node(RemotePid, FromPid, OurDump) -> [RemotePid, length(OurDump)], F). %% Only the node that owns the data could update/remove the data. -%% Key is {USR, Sid} +%% Ideally Key should contain inserter node info (for cleaning). insert(Tab, Rec) -> Servers = other_servers(Tab), ets:insert(Tab, Rec), %% Insert to other nodes and block till written Monitors = insert_to_remote_nodes(Servers, Rec), - wait_for_inserted(Monitors). + wait_for_updated(Monitors). insert_to_remote_nodes([{RemotePid, ProxyPid} | Servers], Rec) -> Mon = erlang:monitor(process, ProxyPid), @@ -103,14 +103,31 @@ insert_to_remote_nodes([{RemotePid, ProxyPid} | Servers], Rec) -> insert_to_remote_nodes([], _Rec) -> []. -wait_for_inserted([Mon | Monitors]) -> +delete(Tab, Key) -> + delete_many(Tab, [Key]). + +%% A separate function for multidelete (because key COULD be a list, so no confusion) +delete_many(Tab, Keys) -> + Servers = other_servers(Tab), + ets_delete_keys(Tab, Keys), + Monitors = delete_from_remote_nodes(Servers, Keys), + wait_for_updated(Monitors). + +delete_from_remote_nodes([{RemotePid, ProxyPid} | Servers], Keys) -> + Mon = erlang:monitor(process, ProxyPid), + erlang:send(RemotePid, {delete_from_remote_node, Mon, self(), Keys}, [noconnect]), + [Mon | delete_from_remote_nodes(Servers, Keys)]; +delete_from_remote_nodes([], _Keys) -> + []. + +wait_for_updated([Mon | Monitors]) -> receive - {inserted, Mon2} when Mon2 =:= Mon -> - wait_for_inserted(Monitors); + {updated, Mon2} when Mon2 =:= Mon -> + wait_for_updated(Monitors); {'DOWN', Mon2, process, _Pid, _Reason} when Mon2 =:= Mon -> - wait_for_inserted(Monitors) + wait_for_updated(Monitors) end; -wait_for_inserted([]) -> +wait_for_updated([]) -> ok. other_servers(Tab) -> @@ -147,7 +164,11 @@ handle_info({'DOWN', _Mon, process, Pid, _Reason}, State) -> handle_down(Pid, State); handle_info({insert_from_remote_node, Mon, Pid, Rec}, State = #{tab := Tab}) -> ets:insert(Tab, Rec), - Pid ! {inserted, Mon}, + Pid ! {updated, Mon}, + {noreply, State}; +handle_info({delete_from_remote_node, Mon, Pid, Keys}, State = #{tab := Tab}) -> + ets_delete_keys(Tab, Keys), + Pid ! {updated, Mon}, {noreply, State}. terminate(_Reason, _State = #{tab := Tab}) -> @@ -245,6 +266,12 @@ update_pt(Tab, Servers2) -> pids_to_nodes(Pids) -> lists:map(fun node/1, Pids). +ets_delete_keys(Tab, [Key | Keys]) -> + ets:delete(Tab, Key), + ets_delete_keys(Tab, Keys); +ets_delete_keys(_Tab, []) -> + ok. + %% Cleanup call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> case Opts of diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index 80e7aa9e..0b4558cc 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -38,22 +38,33 @@ test_multinode(Config) -> insert(Node2, Tab, {b}), insert(Node3, Tab, {c}), insert(Node4, Tab, {d}), - [{a},{c}] = dump(Node1, Tab), - [{b},{d}] = dump(Node2, Tab), + [{a}, {c}] = dump(Node1, Tab), + [{b}, {d}] = dump(Node2, Tab), join(Node1, Pid2, Tab), - [{a},{b},{c},{d}] = dump(Node1, Tab), - [{a},{b},{c},{d}] = dump(Node2, Tab), + [{a}, {b}, {c}, {d}] = dump(Node1, Tab), + [{a}, {b}, {c}, {d}] = dump(Node2, Tab), insert(Node1, Tab, {f}), insert(Node4, Tab, {e}), - AF = [{a},{b},{c},{d},{e},{f}], - AF = dump(Node1, Tab), - AF = dump(Node2, Tab), - AF = dump(Node3, Tab), - AF = dump(Node4, Tab), + Same = fun(X) -> + X = dump(Node1, Tab), + X = dump(Node2, Tab), + X = dump(Node3, Tab), + X = dump(Node4, Tab) + end, + Same([{a}, {b}, {c}, {d}, {e}, {f}]), [Node2, Node3, Node4] = other_nodes(Node1, Tab), [Node1, Node3, Node4] = other_nodes(Node2, Tab), [Node1, Node2, Node4] = other_nodes(Node3, Tab), [Node1, Node2, Node3] = other_nodes(Node4, Tab), + delete(Node1, Tab, e), + Same([{a}, {b}, {c}, {d}, {f}]), + delete(Node4, Tab, a), + Same([{b}, {c}, {d}, {f}]), + %% Bulk operations are supported + insert(Node4, Tab, [{m}, {a}, {n}, {y}]), + Same([{a}, {b}, {c}, {d}, {f}, {m}, {n}, {y}]), + delete_many(Node4, Tab, [a,n]), + Same([{b}, {c}, {d}, {f}, {m}, {y}]), ok. test_multinode_auto_discovery(Config) -> @@ -102,6 +113,12 @@ start(Node, Tab) -> insert(Node, Tab, Rec) -> rpc(Node, kiss, insert, [Tab, Rec]). +delete(Node, Tab, Key) -> + rpc(Node, kiss, delete, [Tab, Key]). + +delete_many(Node, Tab, Keys) -> + rpc(Node, kiss, delete_many, [Tab, Keys]). + dump(Node, Tab) -> rpc(Node, kiss, dump, [Tab]). From bcc77c328848822f7c2f316449a53e5295ec9b14 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 22 Feb 2022 21:47:16 +0100 Subject: [PATCH 08/64] Allow to add tables to disco dynamically Register name for disco --- src/kiss_discovery.erl | 50 +++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/src/kiss_discovery.erl b/src/kiss_discovery.erl index 62b6ffd6..3139f8f3 100644 --- a/src/kiss_discovery.erl +++ b/src/kiss_discovery.erl @@ -5,7 +5,7 @@ %% So, we use a file with nodes to connect as a discovery mechanism %% (so, you can hardcode nodes or use your method of filling it) -module(kiss_discovery). --export([start/1]). +-export([start/1, start_link/1, add_table/2]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). @@ -13,13 +13,38 @@ %% disco_file %% tables -start(Opts = #{disco_file := _, tables := _}) -> - gen_server:start(?MODULE, [Opts], []). +start(Opts) -> + start_common(start, Opts). + +start_link(Opts) -> + start_common(start_link, Opts). + +start_common(F, Opts = #{disco_file := _}) -> + Args = + case Opts of + #{name := Name} -> + [{local, Name}, ?MODULE, [Opts], []]; + _ -> + [?MODULE, [Opts], []] + end, + apply(gen_server, F, Args). + +add_table(Server, Table) -> + gen_server:call(Server, {add_table, Table}). init([Opts]) -> self() ! check, - {ok, Opts#{results => []}}. + Tables = maps:get(tables, Opts, []), + {ok, Opts#{results => [], tables => Tables}}. +handle_call({add_table, Table}, _From, State = #{tables := Tables}) -> + case lists:member(Table, Tables) of + true -> + {reply, {error, already_added}, State}; + false -> + State2 = State#{tables => [Table | Tables]}, + {reply, ok, handle_check(State2)} + end; handle_call(_Reply, _From, State) -> {reply, ok, State}. @@ -35,6 +60,9 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +handle_check(State = #{tables := []}) -> + %% No tables to track, skip + schedule_check(State); handle_check(State = #{disco_file := Filename, tables := Tables}) -> State2 = case file:read_file(Filename) of {error, Reason} -> @@ -48,11 +76,17 @@ handle_check(State = #{disco_file := Filename, tables := Tables}) -> report_results(Results, State), State#{results => Results} end, - schedule_check(), - State2. + schedule_check(State2). -schedule_check() -> - erlang:send_after(5000, self(), check). +schedule_check(State) -> + case State of + #{timer_ref := OldRef} -> + erlang:cancel_timer(OldRef); + _ -> + ok + end, + TimerRef = erlang:send_after(5000, self(), check), + State#{timer_ref => TimerRef}. do_join(Tab, Node) -> From 00b5f2f9e7127f34085c62a52db1c0b81126cea2 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 23 Feb 2022 13:46:56 +0100 Subject: [PATCH 09/64] Use nosuspend when sending messages --- src/kiss.erl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 5389fda0..18947106 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -98,7 +98,7 @@ insert(Tab, Rec) -> insert_to_remote_nodes([{RemotePid, ProxyPid} | Servers], Rec) -> Mon = erlang:monitor(process, ProxyPid), - erlang:send(RemotePid, {insert_from_remote_node, Mon, self(), Rec}, [noconnect]), + erlang:send(RemotePid, {insert_from_remote_node, Mon, self(), Rec}, [noconnect, nosuspend]), [Mon | insert_to_remote_nodes(Servers, Rec)]; insert_to_remote_nodes([], _Rec) -> []. @@ -115,7 +115,7 @@ delete_many(Tab, Keys) -> delete_from_remote_nodes([{RemotePid, ProxyPid} | Servers], Keys) -> Mon = erlang:monitor(process, ProxyPid), - erlang:send(RemotePid, {delete_from_remote_node, Mon, self(), Keys}, [noconnect]), + erlang:send(RemotePid, {delete_from_remote_node, Mon, self(), Keys}, [noconnect, nosuspend]), [Mon | delete_from_remote_nodes(Servers, Keys)]; delete_from_remote_nodes([], _Keys) -> []. @@ -164,11 +164,11 @@ handle_info({'DOWN', _Mon, process, Pid, _Reason}, State) -> handle_down(Pid, State); handle_info({insert_from_remote_node, Mon, Pid, Rec}, State = #{tab := Tab}) -> ets:insert(Tab, Rec), - Pid ! {updated, Mon}, + reply_updated(Pid, Mon), {noreply, State}; handle_info({delete_from_remote_node, Mon, Pid, Keys}, State = #{tab := Tab}) -> ets_delete_keys(Tab, Keys), - Pid ! {updated, Mon}, + reply_updated(Pid, Mon), {noreply, State}. terminate(_Reason, _State = #{tab := Tab}) -> @@ -288,3 +288,7 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> _ -> ok end. + +reply_updated(Pid, Mon) -> + %% We really don't wanna block this process + erlang:send(Pid, {updated, Mon}, [noconnect, nosuspend]). From 172bd79a4a6c8bb51435a7a513acebff8e992e9b Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Mar 2022 16:26:30 +0100 Subject: [PATCH 10/64] Use run_safely Remove try-catch --- src/kiss.erl | 55 +++++++++++++++++++++++------------------------ src/kiss_long.erl | 12 +++++++++++ 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 18947106..350059ed 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -51,6 +51,9 @@ join(LockKey, Tab, RemotePid) when is_pid(RemotePid) -> end. join_loop(LockKey, Tab, RemotePid, Start) -> + %% Only one join at a time: + %% - for performance reasons, we don't want to cause too much load for active nodes + %% - to avoid deadlocks, because joining does gen_server calls F = fun() -> Diff = timer:now_diff(os:timestamp(), Start) div 1000, %% Getting the lock could take really long time in case nodes are @@ -72,20 +75,20 @@ join_loop(LockKey, Tab, RemotePid, Start) -> remote_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> F = fun() -> gen_server:call(RemotePid, {remote_add_node_to_schema, ServerPid, OtherPids}, infinity) end, - kiss_long:run("task=remote_add_node_to_schema remote_pid=~p remote_node=~p other_pids=~0p other_nodes=~0p ", - [RemotePid, node(RemotePid), OtherPids, pids_to_nodes(OtherPids)], F). + kiss_long:run_safely("task=remote_add_node_to_schema remote_pid=~p remote_node=~p other_pids=~0p other_nodes=~0p ", + [RemotePid, node(RemotePid), OtherPids, pids_to_nodes(OtherPids)], F). remote_just_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> F = fun() -> gen_server:call(RemotePid, {remote_just_add_node_to_schema, ServerPid, OtherPids}, infinity) end, - kiss_long:run("task=remote_just_add_node_to_schema remote_pid=~p remote_node=~p other_pids=~0p other_nodes=~0p ", - [RemotePid, node(RemotePid), OtherPids, pids_to_nodes(OtherPids)], F). + kiss_long:run_safely("task=remote_just_add_node_to_schema remote_pid=~p remote_node=~p other_pids=~0p other_nodes=~0p ", + [RemotePid, node(RemotePid), OtherPids, pids_to_nodes(OtherPids)], F). send_dump_to_remote_node(_RemotePid, _FromPid, []) -> skipped; send_dump_to_remote_node(RemotePid, FromPid, OurDump) -> F = fun() -> gen_server:call(RemotePid, {send_dump_to_remote_node, FromPid, OurDump}, infinity) end, - kiss_long:run("task=send_dump_to_remote_node remote_pid=~p count=~p ", - [RemotePid, length(OurDump)], F). + kiss_long:run_safely("task=send_dump_to_remote_node remote_pid=~p count=~p ", + [RemotePid, length(OurDump)], F). %% Only the node that owns the data could update/remove the data. %% Ideally Key should contain inserter node info (for cleaning). @@ -122,16 +125,15 @@ delete_from_remote_nodes([], _Keys) -> wait_for_updated([Mon | Monitors]) -> receive - {updated, Mon2} when Mon2 =:= Mon -> + {updated, Mon} -> wait_for_updated(Monitors); - {'DOWN', Mon2, process, _Pid, _Reason} when Mon2 =:= Mon -> + {'DOWN', Mon, process, _Pid, _Reason} -> wait_for_updated(Monitors) end; wait_for_updated([]) -> ok. other_servers(Tab) -> -% gen_server:call(Tab, get_other_servers). kiss_pt:get(Tab). other_nodes(Tab) -> @@ -151,8 +153,6 @@ handle_call({remote_just_add_node_to_schema, ServerPid, OtherPids}, _From, State handle_remote_just_add_node_to_schema(ServerPid, OtherPids, State); handle_call({send_dump_to_remote_node, FromPid, Dump}, _From, State) -> handle_send_dump_to_remote_node(FromPid, Dump, State); -handle_call(get_other_servers, _From, State = #{other_servers := Servers}) -> - {reply, Servers, State}; handle_call({insert, Rec}, _From, State = #{tab := Tab}) -> ets:insert(Tab, Rec), {reply, ok, State}. @@ -186,20 +186,22 @@ handle_join(RemotePid, State = #{tab := Tab, other_servers := Servers}) when is_ {reply, ok, State}; false -> KnownPids = [Pid || {Pid, _} <- Servers], - %% TODO can crash - case remote_add_node_to_schema(RemotePid, self(), KnownPids) of + Self = self(), + %% Remote gen_server calls here are "safe" + case remote_add_node_to_schema(RemotePid, Self, KnownPids) of {ok, Dump, OtherPids} -> + NewPids = [RemotePid | OtherPids], %% Let all nodes to know each other - [remote_just_add_node_to_schema(Pid, self(), KnownPids) || Pid <- OtherPids], - [remote_just_add_node_to_schema(Pid, self(), [RemotePid | OtherPids]) || Pid <- KnownPids], - Servers2 = lists:usort(start_proxies_for([RemotePid | OtherPids], Servers) ++ Servers), + [remote_just_add_node_to_schema(Pid, Self, KnownPids) || Pid <- OtherPids], + [remote_just_add_node_to_schema(Pid, Self, NewPids) || Pid <- KnownPids], + Servers2 = add_servers(NewPids, Servers), %% Ask our node to replicate data there before applying the dump update_pt(Tab, Servers2), OurDump = dump(Tab), %% Send to all nodes from that partition - [send_dump_to_remote_node(Pid, self(), OurDump) || Pid <- [RemotePid | OtherPids]], + [send_dump_to_remote_node(Pid, Self, OurDump) || Pid <- NewPids], %% Apply to our nodes - [send_dump_to_remote_node(Pid, self(), Dump) || Pid <- KnownPids], + [send_dump_to_remote_node(Pid, Self, Dump) || Pid <- KnownPids], insert_many(Tab, Dump), %% Add ourself into remote schema %% Add remote nodes into our schema @@ -220,11 +222,14 @@ handle_remote_add_node_to_schema(ServerPid, OtherPids, State = #{tab := Tab}) -> end. handle_remote_just_add_node_to_schema(RemotePid, OtherPids, State = #{tab := Tab, other_servers := Servers}) -> - Servers2 = lists:usort(start_proxies_for([RemotePid | OtherPids], Servers) ++ Servers), + Servers2 = add_servers([RemotePid | OtherPids], Servers), update_pt(Tab, Servers2), KnownPids = [Pid || {Pid, _} <- Servers], {reply, {ok, KnownPids}, State#{other_servers => Servers2}}. +add_servers(Pids, Servers) -> + lists:sort(start_proxies_for(Pids, Servers) ++ Servers). + start_proxies_for([RemotePid | OtherPids], AlreadyAddedNodes) when is_pid(RemotePid), RemotePid =/= self() -> case lists:keymember(RemotePid, 1, AlreadyAddedNodes) of @@ -276,15 +281,9 @@ ets_delete_keys(_Tab, []) -> call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> case Opts of #{handle_down := F} -> - try - FF = fun() -> F(#{remote_pid => RemotePid, table => Tab}) end, - kiss_long:run("task=call_user_handle_down table=~p remote_pid=~p remote_node=~p ", - [Tab, RemotePid, node(RemotePid)], FF) - catch Class:Error:Stacktrace -> - error_logger:error_msg("what=call_user_handle_down_failed table=~p " - "remote_pid=~p remote_node=~p class=~p reason=~0p stacktrace=~0p", - [Tab, RemotePid, node(RemotePid), Class, Error, Stacktrace]) - end; + FF = fun() -> F(#{remote_pid => RemotePid, table => Tab}) end, + kiss_long:run_safely("task=call_user_handle_down table=~p remote_pid=~p remote_node=~p ", + [Tab, RemotePid, node(RemotePid)], FF); _ -> ok end. diff --git a/src/kiss_long.erl b/src/kiss_long.erl index fa0ac1c8..f97d19ce 100644 --- a/src/kiss_long.erl +++ b/src/kiss_long.erl @@ -1,13 +1,25 @@ -module(kiss_long). -export([run/3]). +-export([run_safely/3]). + +run_safely(InfoText, InfoArgs, Fun) -> + run(InfoText, InfoArgs, Fun, true). run(InfoText, InfoArgs, Fun) -> + run(InfoText, InfoArgs, Fun, false). + +run(InfoText, InfoArgs, Fun, Catch) -> Parent = self(), Start = os:timestamp(), error_logger:info_msg("what=long_task_started " ++ InfoText, InfoArgs), Pid = spawn_mon(InfoText, InfoArgs, Parent, Start), try Fun() + catch Class:Reason:Stacktrace when Catch -> + error_logger:info_msg("what=long_task_failed " + "class=~p reason=~0p stacktrace=~0p " ++ InfoText, + [Class, Reason, Stacktrace] ++ InfoArgs), + {error, {Class, Reason, Stacktrace}} after Diff = diff(Start), error_logger:info_msg("what=long_task_finished time=~p ms " ++ InfoText, From 272a81da456903edd1ca83e0b2b0cb70331640e8 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Mar 2022 21:27:37 +0100 Subject: [PATCH 11/64] Support discovery backends --- src/kiss_discovery.erl | 70 ++++++++++++++++++------------------- src/kiss_discovery_file.erl | 24 +++++++++++++ 2 files changed, 58 insertions(+), 36 deletions(-) create mode 100644 src/kiss_discovery_file.erl diff --git a/src/kiss_discovery.erl b/src/kiss_discovery.erl index 3139f8f3..32293650 100644 --- a/src/kiss_discovery.erl +++ b/src/kiss_discovery.erl @@ -1,9 +1,4 @@ -%% AWS autodiscovery is kinda bad. -%% - UDP broadcasts do not work -%% - AWS CLI needs access -%% - DNS does not allow to list subdomains -%% So, we use a file with nodes to connect as a discovery mechanism -%% (so, you can hardcode nodes or use your method of filling it) +%% Joins table together when a new node appears -module(kiss_discovery). -export([start/1, start_link/1, add_table/2]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -11,31 +6,38 @@ -behaviour(gen_server). -%% disco_file -%% tables +-type backend_state() :: term(). +-type get_nodes_result() :: {ok, [node()]} | {error, term()}. + +-callback init(map()) -> backend_state(). +-callback get_nodes(backend_state()) -> {get_nodes_result(), backend_state()}. + start(Opts) -> start_common(start, Opts). start_link(Opts) -> start_common(start_link, Opts). -start_common(F, Opts = #{disco_file := _}) -> +start_common(F, Opts) -> Args = case Opts of #{name := Name} -> - [{local, Name}, ?MODULE, [Opts], []]; + [{local, Name}, ?MODULE, Opts, []]; _ -> - [?MODULE, [Opts], []] + [?MODULE, Opts, []] end, apply(gen_server, F, Args). add_table(Server, Table) -> gen_server:call(Server, {add_table, Table}). -init([Opts]) -> +init(Opts) -> + Mod = maps:get(backend_module, Opts, kiss_discovery_file), self() ! check, Tables = maps:get(tables, Opts, []), - {ok, Opts#{results => [], tables => Tables}}. + BackendState = Mod:init(Opts), + {ok, #{results => [], tables => Tables, + backend_module => Mod, backend_state => BackendState}}. handle_call({add_table, Table}, _From, State = #{tables := Tables}) -> case lists:member(Table, Tables) of @@ -48,7 +50,7 @@ handle_call({add_table, Table}, _From, State = #{tables := Tables}) -> handle_call(_Reply, _From, State) -> {reply, ok, State}. -handle_cast(Msg, State) -> +handle_cast(_Msg, State) -> {noreply, State}. handle_info(check, State) -> @@ -63,31 +65,27 @@ code_change(_OldVsn, State, _Extra) -> handle_check(State = #{tables := []}) -> %% No tables to track, skip schedule_check(State); -handle_check(State = #{disco_file := Filename, tables := Tables}) -> - State2 = case file:read_file(Filename) of - {error, Reason} -> - error_logger:error_msg("what=discovery_failed filename=~0p reason=~0p", - [Filename, Reason]), - State; - {ok, Text} -> - Lines = binary:split(Text, [<<"\r">>, <<"\n">>, <<" ">>], [global]), - Nodes = [binary_to_atom(X, latin1) || X <- Lines, X =/= <<>>], - Results = [do_join(Tab, Node) || Tab <- Tables, Node <- Nodes, node() =/= Node], - report_results(Results, State), - State#{results => Results} - end, - schedule_check(State2). +handle_check(State = #{backend_module := Mod, backend_state := BackendState}) -> + {Res, BackendState2} = Mod:get_nodes(BackendState), + State2 = handle_get_nodes_result(Res, State), + schedule_check(State2#{backend_state => BackendState2}). + +handle_get_nodes_result({error, _Reason}, State) -> + State; +handle_get_nodes_result({ok, Nodes}, State = #{tables := Tables}) -> + Results = [do_join(Tab, Node) || Tab <- Tables, Node <- Nodes, node() =/= Node], + report_results(Results, State), + State#{results => Results}. schedule_check(State) -> - case State of - #{timer_ref := OldRef} -> - erlang:cancel_timer(OldRef); - _ -> - ok - end, + cancel_old_timer(State), TimerRef = erlang:send_after(5000, self(), check), State#{timer_ref => TimerRef}. +cancel_old_timer(#{timer_ref := OldRef}) -> + erlang:cancel_timer(OldRef); +cancel_old_timer(_State) -> + ok. do_join(Tab, Node) -> %% That would trigger autoconnect for the first time @@ -99,9 +97,9 @@ do_join(Tab, Node) -> #{what => pid_not_found, reason => Other, node => Node, table => Tab} end. -report_results(Results, State = #{results := OldResults}) -> +report_results(Results, _State = #{results := OldResults}) -> Changed = Results -- OldResults, - [report_result(Result) || Result <- Results]. + [report_result(Result) || Result <- Changed]. report_result(Map) -> Text = [io_lib:format("~0p=~0p ", [K, V]) || {K, V} <- maps:to_list(Map)], diff --git a/src/kiss_discovery_file.erl b/src/kiss_discovery_file.erl new file mode 100644 index 00000000..f8832100 --- /dev/null +++ b/src/kiss_discovery_file.erl @@ -0,0 +1,24 @@ +%% AWS auto-discovery is kinda bad. +%% - UDP broadcasts do not work +%% - AWS CLI needs access +%% - DNS does not allow to list subdomains +%% So, we use a file with nodes to connect as a discovery mechanism +%% (so, you can hardcode nodes or use your method of filling it) +-module(kiss_discovery_file). +-behaviour(kiss_discovery). +-export([init/1, get_nodes/1]). + +init(Opts) -> + Opts. + +get_nodes(State = #{disco_file := Filename}) -> + case file:read_file(Filename) of + {error, Reason} -> + error_logger:error_msg("what=discovery_failed filename=~0p reason=~0p", + [Filename, Reason]), + {{error, Reason}, State}; + {ok, Text} -> + Lines = binary:split(Text, [<<"\r">>, <<"\n">>, <<" ">>], [global]), + Nodes = [binary_to_atom(X, latin1) || X <- Lines, X =/= <<>>], + {{ok, Nodes}, State} + end. From 3472f1396cb895a2773a774c56e462eefe1eae74 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Mar 2022 21:52:47 +0100 Subject: [PATCH 12/64] Use OTP logger --- src/kiss.erl | 50 ++++++++++++++++++++++++------------- src/kiss_discovery.erl | 6 +++-- src/kiss_discovery_file.erl | 6 +++-- src/kiss_long.erl | 46 ++++++++++++++++------------------ src/kiss_proxy.erl | 10 +++++--- 5 files changed, 68 insertions(+), 50 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 350059ed..172eea44 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -11,12 +11,14 @@ %% We don't use monitors to avoid round-trips (that's why we don't use calls neither) -module(kiss). +-behaviour(gen_server). + -export([start/2, stop/1, dump/1, insert/2, delete/2, delete_many/2, join/3, other_nodes/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --behaviour(gen_server). +-include_lib("kernel/include/logger.hrl"). %% Table and server has the same name %% Opts: @@ -46,8 +48,8 @@ join(LockKey, Tab, RemotePid) when is_pid(RemotePid) -> false -> Start = os:timestamp(), F = fun() -> join_loop(LockKey, Tab, RemotePid, Start) end, - kiss_long:run("task=join table=~p remote_pid=~p remote_node=~p ", - [Tab, RemotePid, node(RemotePid)], F) + kiss_long:run(#{task => join, table => Tab, remote_pid => RemotePid, + remote_node => node(RemotePid)}, F) end. join_loop(LockKey, Tab, RemotePid, Start) -> @@ -58,7 +60,7 @@ join_loop(LockKey, Tab, RemotePid, Start) -> Diff = timer:now_diff(os:timestamp(), Start) div 1000, %% Getting the lock could take really long time in case nodes are %% overloaded or joining is already in progress on another node - error_logger:info_msg("what=join_got_lock table=~p after_time=~p ms", [Tab, Diff]), + ?LOG_INFO(#{what => join_got_lock, table => Tab, after_time_ms => Diff}), gen_server:call(Tab, {join, RemotePid}, infinity) end, LockRequest = {LockKey, self()}, @@ -67,7 +69,7 @@ join_loop(LockKey, Tab, RemotePid, Start) -> Retries = 1, case global:trans(LockRequest, F, Nodes, Retries) of aborted -> - error_logger:error_msg("what=join_retry reason=lock_aborted", []), + ?LOG_ERROR(#{what => join_retry, reason => lock_aborted}), join_loop(LockKey, Tab, RemotePid, Start); Result -> Result @@ -75,20 +77,25 @@ join_loop(LockKey, Tab, RemotePid, Start) -> remote_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> F = fun() -> gen_server:call(RemotePid, {remote_add_node_to_schema, ServerPid, OtherPids}, infinity) end, - kiss_long:run_safely("task=remote_add_node_to_schema remote_pid=~p remote_node=~p other_pids=~0p other_nodes=~0p ", - [RemotePid, node(RemotePid), OtherPids, pids_to_nodes(OtherPids)], F). + Info = #{task => remote_add_node_to_schema, + remote_pid => RemotePid, remote_node => node(RemotePid), + other_pids => OtherPids, other_nodes => pids_to_nodes(OtherPids)}, + kiss_long:run_safely(Info, F). remote_just_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> F = fun() -> gen_server:call(RemotePid, {remote_just_add_node_to_schema, ServerPid, OtherPids}, infinity) end, - kiss_long:run_safely("task=remote_just_add_node_to_schema remote_pid=~p remote_node=~p other_pids=~0p other_nodes=~0p ", - [RemotePid, node(RemotePid), OtherPids, pids_to_nodes(OtherPids)], F). + Info = #{task => remote_just_add_node_to_schema, + remote_pid => RemotePid, remote_node => node(RemotePid), + other_pids => OtherPids, other_nodes => pids_to_nodes(OtherPids)}, + kiss_long:run_safely(Info, F). send_dump_to_remote_node(_RemotePid, _FromPid, []) -> skipped; send_dump_to_remote_node(RemotePid, FromPid, OurDump) -> F = fun() -> gen_server:call(RemotePid, {send_dump_to_remote_node, FromPid, OurDump}, infinity) end, - kiss_long:run_safely("task=send_dump_to_remote_node remote_pid=~p count=~p ", - [RemotePid, length(OurDump)], F). + Info = #{task => send_dump_to_remote_node, + remote_pid => RemotePid, count => length(OurDump)}, + kiss_long:run_safely(Info, F). %% Only the node that owns the data could update/remove the data. %% Ideally Key should contain inserter node info (for cleaning). @@ -101,7 +108,8 @@ insert(Tab, Rec) -> insert_to_remote_nodes([{RemotePid, ProxyPid} | Servers], Rec) -> Mon = erlang:monitor(process, ProxyPid), - erlang:send(RemotePid, {insert_from_remote_node, Mon, self(), Rec}, [noconnect, nosuspend]), + Msg = {insert_from_remote_node, Mon, self(), Rec}, + send_to_remote(RemotePid, Msg), [Mon | insert_to_remote_nodes(Servers, Rec)]; insert_to_remote_nodes([], _Rec) -> []. @@ -118,11 +126,15 @@ delete_many(Tab, Keys) -> delete_from_remote_nodes([{RemotePid, ProxyPid} | Servers], Keys) -> Mon = erlang:monitor(process, ProxyPid), - erlang:send(RemotePid, {delete_from_remote_node, Mon, self(), Keys}, [noconnect, nosuspend]), + Msg = {delete_from_remote_node, Mon, self(), Keys}, + send_to_remote(RemotePid, Msg), [Mon | delete_from_remote_nodes(Servers, Keys)]; delete_from_remote_nodes([], _Keys) -> []. +send_to_remote(RemotePid, Msg) -> + erlang:send(RemotePid, Msg, [noconnect, nosuspend]). + wait_for_updated([Mon | Monitors]) -> receive {updated, Mon} -> @@ -208,7 +220,7 @@ handle_join(RemotePid, State = #{tab := Tab, other_servers := Servers}) when is_ %% Copy from our node / Copy into our node {reply, ok, State#{other_servers => Servers2}}; Other -> - error_logger:error_msg("remote_add_node_to_schema failed ~p", [Other]), + ?LOG_ERROR(#{what => remote_add_node_to_schema, reason => Other}), {reply, {error, remote_add_node_to_schema_failed}, State} end end. @@ -238,7 +250,8 @@ start_proxies_for([RemotePid | OtherPids], AlreadyAddedNodes) erlang:monitor(process, ProxyPid), [{RemotePid, ProxyPid} | start_proxies_for(OtherPids, AlreadyAddedNodes)]; true -> - error_logger:info_msg("what=already_added remote_pid=~p node=~p", [RemotePid, node(RemotePid)]), + ?LOG_INFO(#{what => already_added, + remote_pid => RemotePid, remote_node => node(RemotePid)}), start_proxies_for(OtherPids, AlreadyAddedNodes) end; start_proxies_for([], _AlreadyAddedNodes) -> @@ -260,7 +273,7 @@ handle_down(ProxyPid, State = #{tab := Tab, other_servers := Servers}) -> {noreply, State#{other_servers => Servers2}}; false -> %% This should not happen - error_logger:error_msg("handle_down failed proxy_pid=~p state=~0p", [ProxyPid, State]), + ?LOG_ERROR(#{what => handle_down_failed, proxy_pid => ProxyPid, state => State}), {noreply, State} end. @@ -282,8 +295,9 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> case Opts of #{handle_down := F} -> FF = fun() -> F(#{remote_pid => RemotePid, table => Tab}) end, - kiss_long:run_safely("task=call_user_handle_down table=~p remote_pid=~p remote_node=~p ", - [Tab, RemotePid, node(RemotePid)], FF); + Info = #{task => call_user_handle_down, table => Tab, + remote_pid => RemotePid, remote_node => node(RemotePid)}, + kiss_long:run_safely(Info, FF); _ -> ok end. diff --git a/src/kiss_discovery.erl b/src/kiss_discovery.erl index 32293650..bf4e2549 100644 --- a/src/kiss_discovery.erl +++ b/src/kiss_discovery.erl @@ -1,10 +1,12 @@ %% Joins table together when a new node appears -module(kiss_discovery). +-behaviour(gen_server). + -export([start/1, start_link/1, add_table/2]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --behaviour(gen_server). +-include_lib("kernel/include/logger.hrl"). -type backend_state() :: term(). -type get_nodes_result() :: {ok, [node()]} | {error, term()}. @@ -103,4 +105,4 @@ report_results(Results, _State = #{results := OldResults}) -> report_result(Map) -> Text = [io_lib:format("~0p=~0p ", [K, V]) || {K, V} <- maps:to_list(Map)], - error_logger:info_msg("discovery ~s", [Text]). + ?LOG_INFO(#{what => discovery_change, result => Text}). diff --git a/src/kiss_discovery_file.erl b/src/kiss_discovery_file.erl index f8832100..2c461763 100644 --- a/src/kiss_discovery_file.erl +++ b/src/kiss_discovery_file.erl @@ -8,14 +8,16 @@ -behaviour(kiss_discovery). -export([init/1, get_nodes/1]). +-include_lib("kernel/include/logger.hrl"). + init(Opts) -> Opts. get_nodes(State = #{disco_file := Filename}) -> case file:read_file(Filename) of {error, Reason} -> - error_logger:error_msg("what=discovery_failed filename=~0p reason=~0p", - [Filename, Reason]), + ?LOG_ERROR(#{what => discovery_failed, + filename => Filename, reason => Reason}), {{error, Reason}, State}; {ok, Text} -> Lines = binary:split(Text, [<<"\r">>, <<"\n">>, <<" ">>], [global]), diff --git a/src/kiss_long.erl b/src/kiss_long.erl index f97d19ce..5ca56cd3 100644 --- a/src/kiss_long.erl +++ b/src/kiss_long.erl @@ -1,51 +1,49 @@ -module(kiss_long). --export([run/3]). --export([run_safely/3]). +-export([run/2]). +-export([run_safely/2]). -run_safely(InfoText, InfoArgs, Fun) -> - run(InfoText, InfoArgs, Fun, true). +-include_lib("kernel/include/logger.hrl"). -run(InfoText, InfoArgs, Fun) -> - run(InfoText, InfoArgs, Fun, false). +run_safely(Info, Fun) -> + run(Info, Fun, true). -run(InfoText, InfoArgs, Fun, Catch) -> +run(Info, Fun) -> + run(Info, Fun, false). + +run(Info, Fun, Catch) -> Parent = self(), Start = os:timestamp(), - error_logger:info_msg("what=long_task_started " ++ InfoText, InfoArgs), - Pid = spawn_mon(InfoText, InfoArgs, Parent, Start), + ?LOG_INFO(Info#{what => long_task_started}), + Pid = spawn_mon(Info, Parent, Start), try Fun() catch Class:Reason:Stacktrace when Catch -> - error_logger:info_msg("what=long_task_failed " - "class=~p reason=~0p stacktrace=~0p " ++ InfoText, - [Class, Reason, Stacktrace] ++ InfoArgs), + ?LOG_INFO(Info#{what => long_task_failed, class => Class, + reason => Reason, stacktrace => Stacktrace}), {error, {Class, Reason, Stacktrace}} after Diff = diff(Start), - error_logger:info_msg("what=long_task_finished time=~p ms " ++ InfoText, - [Diff] ++ InfoArgs), + ?LOG_INFO(Info#{what => long_task_finished, time_ms => Diff}), Pid ! stop end. -spawn_mon(InfoText, InfoArgs, Parent, Start) -> - spawn_link(fun() -> run_monitor(InfoText, InfoArgs, Parent, Start) end). +spawn_mon(Info, Parent, Start) -> + spawn_link(fun() -> run_monitor(Info, Parent, Start) end). -run_monitor(InfoText, InfoArgs, Parent, Start) -> +run_monitor(Info, Parent, Start) -> Mon = erlang:monitor(process, Parent), - monitor_loop(Mon, InfoText, InfoArgs, Start). + monitor_loop(Mon, Info, Start). -monitor_loop(Mon, InfoText, InfoArgs, Start) -> +monitor_loop(Mon, Info, Start) -> receive {'DOWN', MonRef, process, _Pid, Reason} when Mon =:= MonRef -> - error_logger:error_msg("what=long_task_failed reason=~p " ++ InfoText, - [Reason] ++ InfoArgs), + ?LOG_ERROR(Info#{what => long_task_failed, reason => Reason}), ok; stop -> ok after 5000 -> Diff = diff(Start), - error_logger:info_msg("what=long_task_progress time=~p ms " ++ InfoText, - [Diff] ++ InfoArgs), - monitor_loop(Mon, InfoText, InfoArgs, Start) + ?LOG_INFO(Info#{what => long_task_progress, time_ms => Diff}), + monitor_loop(Mon, Info, Start) end. diff(Start) -> diff --git a/src/kiss_proxy.erl b/src/kiss_proxy.erl index d76dee4c..aacca0af 100644 --- a/src/kiss_proxy.erl +++ b/src/kiss_proxy.erl @@ -1,10 +1,12 @@ %% We monitor this process instead of a remote process -module(kiss_proxy). +-behaviour(gen_server). + -export([start/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --behaviour(gen_server). +-include_lib("kernel/include/logger.hrl"). start(RemotePid) -> gen_server:start(?MODULE, [RemotePid, self()], []). @@ -17,14 +19,14 @@ init([RemotePid, ParentPid]) -> handle_call(_Reply, _From, State) -> {reply, ok, State}. -handle_cast(Msg, State) -> +handle_cast(_Msg, State) -> {noreply, State}. handle_info({'DOWN', MonRef, process, Pid, _Reason}, State = #{mon := MonRef}) -> - error_logger:error_msg("Node down ~p", [node(Pid)]), + ?LOG_ERROR(#{what => node_down, remote_pid => Pid, node => node(Pid)}), {stop, State}; handle_info({'DOWN', MonRef, process, Pid, _Reason}, State = #{pmon := MonRef}) -> - error_logger:error_msg("Parent down ~p", [Pid]), + ?LOG_ERROR(#{what => parent_process_down, parent_pid => Pid}), {stop, State}. terminate(_Reason, _State) -> From d97032e93613069b832710d7f077b39cf5facb68 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Mar 2022 22:29:42 +0100 Subject: [PATCH 13/64] Remove remote_just_add_node_to_schema function --- src/kiss.erl | 92 ++++++++++++++++++++-------------------------------- 1 file changed, 36 insertions(+), 56 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 172eea44..16ac7fd3 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -75,16 +75,10 @@ join_loop(LockKey, Tab, RemotePid, Start) -> Result end. -remote_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> - F = fun() -> gen_server:call(RemotePid, {remote_add_node_to_schema, ServerPid, OtherPids}, infinity) end, - Info = #{task => remote_add_node_to_schema, - remote_pid => RemotePid, remote_node => node(RemotePid), - other_pids => OtherPids, other_nodes => pids_to_nodes(OtherPids)}, - kiss_long:run_safely(Info, F). - -remote_just_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> - F = fun() -> gen_server:call(RemotePid, {remote_just_add_node_to_schema, ServerPid, OtherPids}, infinity) end, - Info = #{task => remote_just_add_node_to_schema, +remote_add_node_to_schema(RemotePid, ServerPid, OtherPids, ReturnDump) -> + Msg = {remote_add_node_to_schema, ServerPid, OtherPids, ReturnDump}, + F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, + Info = #{task => remote_add_node_to_schema, return_dump => ReturnDump, remote_pid => RemotePid, remote_node => node(RemotePid), other_pids => OtherPids, other_nodes => pids_to_nodes(OtherPids)}, kiss_long:run_safely(Info, F). @@ -92,7 +86,8 @@ remote_just_add_node_to_schema(RemotePid, ServerPid, OtherPids) -> send_dump_to_remote_node(_RemotePid, _FromPid, []) -> skipped; send_dump_to_remote_node(RemotePid, FromPid, OurDump) -> - F = fun() -> gen_server:call(RemotePid, {send_dump_to_remote_node, FromPid, OurDump}, infinity) end, + Msg = {send_dump_to_remote_node, FromPid, OurDump}, + F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, Info = #{task => send_dump_to_remote_node, remote_pid => RemotePid, count => length(OurDump)}, kiss_long:run_safely(Info, F). @@ -154,15 +149,13 @@ other_nodes(Tab) -> init([Tab, Opts]) -> ets:new(Tab, [ordered_set, named_table, public, {read_concurrency, true}]), - update_pt(Tab, []), + kiss_pt:put(Tab, []), {ok, #{tab => Tab, other_servers => [], opts => Opts}}. handle_call({join, RemotePid}, _From, State) -> handle_join(RemotePid, State); -handle_call({remote_add_node_to_schema, ServerPid, OtherPids}, _From, State) -> - handle_remote_add_node_to_schema(ServerPid, OtherPids, State); -handle_call({remote_just_add_node_to_schema, ServerPid, OtherPids}, _From, State) -> - handle_remote_just_add_node_to_schema(ServerPid, OtherPids, State); +handle_call({remote_add_node_to_schema, ServerPid, OtherPids, ReturnDump}, _From, State) -> + handle_remote_add_node_to_schema(ServerPid, OtherPids, ReturnDump, State); handle_call({send_dump_to_remote_node, FromPid, Dump}, _From, State) -> handle_send_dump_to_remote_node(FromPid, Dump, State); handle_call({insert, Rec}, _From, State = #{tab := Tab}) -> @@ -200,21 +193,21 @@ handle_join(RemotePid, State = #{tab := Tab, other_servers := Servers}) when is_ KnownPids = [Pid || {Pid, _} <- Servers], Self = self(), %% Remote gen_server calls here are "safe" - case remote_add_node_to_schema(RemotePid, Self, KnownPids) of + case remote_add_node_to_schema(RemotePid, Self, KnownPids, true) of {ok, Dump, OtherPids} -> NewPids = [RemotePid | OtherPids], %% Let all nodes to know each other - [remote_just_add_node_to_schema(Pid, Self, KnownPids) || Pid <- OtherPids], - [remote_just_add_node_to_schema(Pid, Self, NewPids) || Pid <- KnownPids], + [remote_add_node_to_schema(Pid, Self, KnownPids, false) || Pid <- OtherPids], + [remote_add_node_to_schema(Pid, Self, NewPids, false) || Pid <- KnownPids], Servers2 = add_servers(NewPids, Servers), %% Ask our node to replicate data there before applying the dump - update_pt(Tab, Servers2), + kiss_pt:put(Tab, Servers2), OurDump = dump(Tab), %% Send to all nodes from that partition [send_dump_to_remote_node(Pid, Self, OurDump) || Pid <- NewPids], %% Apply to our nodes [send_dump_to_remote_node(Pid, Self, Dump) || Pid <- KnownPids], - insert_many(Tab, Dump), + ets:insert(Tab, Dump), %% Add ourself into remote schema %% Add remote nodes into our schema %% Copy from our node / Copy into our node @@ -225,19 +218,30 @@ handle_join(RemotePid, State = #{tab := Tab, other_servers := Servers}) when is_ end end. -handle_remote_add_node_to_schema(ServerPid, OtherPids, State = #{tab := Tab}) -> - case handle_remote_just_add_node_to_schema(ServerPid, OtherPids, State) of - {reply, {ok, KnownPids}, State2} -> - {reply, {ok, dump(Tab), KnownPids}, State2}; - Other -> - Other - end. - -handle_remote_just_add_node_to_schema(RemotePid, OtherPids, State = #{tab := Tab, other_servers := Servers}) -> +handle_remote_add_node_to_schema(RemotePid, OtherPids, ReturnDump, + State = #{tab := Tab, other_servers := Servers}) -> Servers2 = add_servers([RemotePid | OtherPids], Servers), - update_pt(Tab, Servers2), + kiss_pt:put(Tab, Servers2), KnownPids = [Pid || {Pid, _} <- Servers], - {reply, {ok, KnownPids}, State#{other_servers => Servers2}}. + Dump = case ReturnDump of true -> dump(Tab); false -> not_requested end, + {reply, {ok, Dump, KnownPids}, State#{other_servers => Servers2}}. + +handle_send_dump_to_remote_node(_FromPid, Dump, State = #{tab := Tab}) -> + ets:insert(Tab, Dump), + {reply, ok, State}. + +handle_down(ProxyPid, State = #{tab := Tab, other_servers := Servers}) -> + case lists:keytake(ProxyPid, 2, Servers) of + {value, {RemotePid, _}, Servers2} -> + %% Down from a proxy + kiss_pt:put(Tab, Servers2), + call_user_handle_down(RemotePid, State), + {noreply, State#{other_servers => Servers2}}; + false -> + %% This should not happen + ?LOG_ERROR(#{what => handle_down_failed, proxy_pid => ProxyPid, state => State}), + {noreply, State} + end. add_servers(Pids, Servers) -> lists:sort(start_proxies_for(Pids, Servers) ++ Servers). @@ -257,30 +261,6 @@ start_proxies_for([RemotePid | OtherPids], AlreadyAddedNodes) start_proxies_for([], _AlreadyAddedNodes) -> []. -handle_send_dump_to_remote_node(_FromPid, Dump, State = #{tab := Tab}) -> - insert_many(Tab, Dump), - {reply, ok, State}. - -insert_many(Tab, Recs) -> - ets:insert(Tab, Recs). - -handle_down(ProxyPid, State = #{tab := Tab, other_servers := Servers}) -> - case lists:keytake(ProxyPid, 2, Servers) of - {value, {RemotePid, _}, Servers2} -> - %% Down from a proxy - update_pt(Tab, Servers2), - call_user_handle_down(RemotePid, State), - {noreply, State#{other_servers => Servers2}}; - false -> - %% This should not happen - ?LOG_ERROR(#{what => handle_down_failed, proxy_pid => ProxyPid, state => State}), - {noreply, State} - end. - -%% Called each time other_servers changes -update_pt(Tab, Servers2) -> - kiss_pt:put(Tab, Servers2). - pids_to_nodes(Pids) -> lists:map(fun node/1, Pids). From 60ed57dc019185614f61e3543b297d75c3363170 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 9 May 2022 12:39:27 +0200 Subject: [PATCH 14/64] Add servers_to_pids and has_remote_pid helper functions --- src/kiss.erl | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 16ac7fd3..dc936f8d 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -48,7 +48,8 @@ join(LockKey, Tab, RemotePid) when is_pid(RemotePid) -> false -> Start = os:timestamp(), F = fun() -> join_loop(LockKey, Tab, RemotePid, Start) end, - kiss_long:run(#{task => join, table => Tab, remote_pid => RemotePid, + kiss_long:run(#{task => join, table => Tab, + remote_pid => RemotePid, remote_node => node(RemotePid)}, F) end. @@ -144,7 +145,7 @@ other_servers(Tab) -> kiss_pt:get(Tab). other_nodes(Tab) -> - lists:sort([node(Pid) || {Pid, _} <- other_servers(Tab)]). + lists:usort(pids_to_nodes(servers_to_pids(other_servers(Tab)))). init([Tab, Opts]) -> ets:new(Tab, [ordered_set, named_table, @@ -185,12 +186,12 @@ code_change(_OldVsn, State, _Extra) -> handle_join(RemotePid, State = #{tab := Tab, other_servers := Servers}) when is_pid(RemotePid) -> - case lists:keymember(RemotePid, 1, Servers) of + case has_remote_pid(RemotePid, Servers) of true -> %% Already added {reply, ok, State}; false -> - KnownPids = [Pid || {Pid, _} <- Servers], + KnownPids = servers_to_pids(Servers), Self = self(), %% Remote gen_server calls here are "safe" case remote_add_node_to_schema(RemotePid, Self, KnownPids, true) of @@ -222,7 +223,7 @@ handle_remote_add_node_to_schema(RemotePid, OtherPids, ReturnDump, State = #{tab := Tab, other_servers := Servers}) -> Servers2 = add_servers([RemotePid | OtherPids], Servers), kiss_pt:put(Tab, Servers2), - KnownPids = [Pid || {Pid, _} <- Servers], + KnownPids = servers_to_pids(Servers), Dump = case ReturnDump of true -> dump(Tab); false -> not_requested end, {reply, {ok, Dump, KnownPids}, State#{other_servers => Servers2}}. @@ -246,19 +247,19 @@ handle_down(ProxyPid, State = #{tab := Tab, other_servers := Servers}) -> add_servers(Pids, Servers) -> lists:sort(start_proxies_for(Pids, Servers) ++ Servers). -start_proxies_for([RemotePid | OtherPids], AlreadyAddedNodes) +start_proxies_for([RemotePid | OtherPids], Servers) when is_pid(RemotePid), RemotePid =/= self() -> - case lists:keymember(RemotePid, 1, AlreadyAddedNodes) of + case has_remote_pid(RemotePid, Servers) of false -> {ok, ProxyPid} = kiss_proxy:start(RemotePid), erlang:monitor(process, ProxyPid), - [{RemotePid, ProxyPid} | start_proxies_for(OtherPids, AlreadyAddedNodes)]; + [{RemotePid, ProxyPid} | start_proxies_for(OtherPids, Servers)]; true -> ?LOG_INFO(#{what => already_added, remote_pid => RemotePid, remote_node => node(RemotePid)}), - start_proxies_for(OtherPids, AlreadyAddedNodes) + start_proxies_for(OtherPids, Servers) end; -start_proxies_for([], _AlreadyAddedNodes) -> +start_proxies_for([], _Servers) -> []. pids_to_nodes(Pids) -> @@ -270,6 +271,12 @@ ets_delete_keys(Tab, [Key | Keys]) -> ets_delete_keys(_Tab, []) -> ok. +servers_to_pids(Servers) -> + [Pid || {Pid, _} <- Servers]. + +has_remote_pid(RemotePid, Servers) -> + lists:keymember(RemotePid, 1, Servers). + %% Cleanup call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> case Opts of From 3dbaf3b23ee806b66eee4d3a07e3d224271ba6b5 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Mar 2022 23:13:28 +0100 Subject: [PATCH 15/64] Split handle_join --- src/kiss.erl | 62 +++++++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index dc936f8d..1c6f4255 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -185,40 +185,48 @@ code_change(_OldVsn, State, _Extra) -> {ok, State}. -handle_join(RemotePid, State = #{tab := Tab, other_servers := Servers}) when is_pid(RemotePid) -> +handle_join(RemotePid, State = #{other_servers := Servers}) when is_pid(RemotePid) -> case has_remote_pid(RemotePid, Servers) of true -> %% Already added {reply, ok, State}; false -> - KnownPids = servers_to_pids(Servers), - Self = self(), - %% Remote gen_server calls here are "safe" - case remote_add_node_to_schema(RemotePid, Self, KnownPids, true) of - {ok, Dump, OtherPids} -> - NewPids = [RemotePid | OtherPids], - %% Let all nodes to know each other - [remote_add_node_to_schema(Pid, Self, KnownPids, false) || Pid <- OtherPids], - [remote_add_node_to_schema(Pid, Self, NewPids, false) || Pid <- KnownPids], - Servers2 = add_servers(NewPids, Servers), - %% Ask our node to replicate data there before applying the dump - kiss_pt:put(Tab, Servers2), - OurDump = dump(Tab), - %% Send to all nodes from that partition - [send_dump_to_remote_node(Pid, Self, OurDump) || Pid <- NewPids], - %% Apply to our nodes - [send_dump_to_remote_node(Pid, Self, Dump) || Pid <- KnownPids], - ets:insert(Tab, Dump), - %% Add ourself into remote schema - %% Add remote nodes into our schema - %% Copy from our node / Copy into our node - {reply, ok, State#{other_servers => Servers2}}; - Other -> - ?LOG_ERROR(#{what => remote_add_node_to_schema, reason => Other}), - {reply, {error, remote_add_node_to_schema_failed}, State} - end + handle_join2(RemotePid, State) end. +handle_join2(RemotePid, State = #{other_servers := Servers}) -> + KnownPids = servers_to_pids(Servers), + %% Remote gen_server calls here are "safe" + case remote_add_node_to_schema(RemotePid, self(), KnownPids, true) of + {ok, Dump, OtherPids} -> + handle_join3(RemotePid, Dump, OtherPids, KnownPids, State); + Other -> + ?LOG_ERROR(#{what => remote_add_node_to_schema, reason => Other}), + {reply, {error, remote_add_node_to_schema_failed}, State} + end. + +handle_join3(RemotePid, Dump, OtherPids, KnownPids, + State = #{tab := Tab, other_servers := Servers}) -> + Self = self(), + NewPids = [RemotePid | OtherPids], + %% Let all nodes to know each other + [remote_add_node_to_schema(Pid, Self, KnownPids, false) || Pid <- OtherPids], + [remote_add_node_to_schema(Pid, Self, NewPids, false) || Pid <- KnownPids], + Servers2 = add_servers(NewPids, Servers), + %% Ask our node to replicate data there before applying the dump + kiss_pt:put(Tab, Servers2), + OurDump = dump(Tab), + %% A race condition is possible: when the remote node inserts a deleted record + %% Send to all nodes from that partition + [send_dump_to_remote_node(Pid, Self, OurDump) || Pid <- NewPids], + %% Apply to our nodes + [send_dump_to_remote_node(Pid, Self, Dump) || Pid <- KnownPids], + ets:insert(Tab, Dump), + %% Add ourself into remote schema + %% Add remote nodes into our schema + %% Copy from our node / Copy into our node + {reply, ok, State#{other_servers => Servers2}}. + handle_remote_add_node_to_schema(RemotePid, OtherPids, ReturnDump, State = #{tab := Tab, other_servers := Servers}) -> Servers2 = add_servers([RemotePid | OtherPids], Servers), From 03b1f963dfb9f9d6b47cbfbb3d76fb8cdb82af22 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 4 Mar 2022 11:16:41 +0100 Subject: [PATCH 16/64] Route deletes/inserts though the local server --- src/kiss.erl | 72 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 1c6f4255..b7c02f92 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -95,42 +95,18 @@ send_dump_to_remote_node(RemotePid, FromPid, OurDump) -> %% Only the node that owns the data could update/remove the data. %% Ideally Key should contain inserter node info (for cleaning). -insert(Tab, Rec) -> - Servers = other_servers(Tab), - ets:insert(Tab, Rec), - %% Insert to other nodes and block till written - Monitors = insert_to_remote_nodes(Servers, Rec), +insert(Server, Rec) -> + {ok, Monitors} = gen_server:call(Server, {insert, Rec}), wait_for_updated(Monitors). -insert_to_remote_nodes([{RemotePid, ProxyPid} | Servers], Rec) -> - Mon = erlang:monitor(process, ProxyPid), - Msg = {insert_from_remote_node, Mon, self(), Rec}, - send_to_remote(RemotePid, Msg), - [Mon | insert_to_remote_nodes(Servers, Rec)]; -insert_to_remote_nodes([], _Rec) -> - []. - delete(Tab, Key) -> delete_many(Tab, [Key]). %% A separate function for multidelete (because key COULD be a list, so no confusion) -delete_many(Tab, Keys) -> - Servers = other_servers(Tab), - ets_delete_keys(Tab, Keys), - Monitors = delete_from_remote_nodes(Servers, Keys), +delete_many(Server, Keys) -> + {ok, Monitors} = gen_server:call(Server, {delete, Keys}), wait_for_updated(Monitors). -delete_from_remote_nodes([{RemotePid, ProxyPid} | Servers], Keys) -> - Mon = erlang:monitor(process, ProxyPid), - Msg = {delete_from_remote_node, Mon, self(), Keys}, - send_to_remote(RemotePid, Msg), - [Mon | delete_from_remote_nodes(Servers, Keys)]; -delete_from_remote_nodes([], _Keys) -> - []. - -send_to_remote(RemotePid, Msg) -> - erlang:send(RemotePid, Msg, [noconnect, nosuspend]). - wait_for_updated([Mon | Monitors]) -> receive {updated, Mon} -> @@ -159,9 +135,10 @@ handle_call({remote_add_node_to_schema, ServerPid, OtherPids, ReturnDump}, _From handle_remote_add_node_to_schema(ServerPid, OtherPids, ReturnDump, State); handle_call({send_dump_to_remote_node, FromPid, Dump}, _From, State) -> handle_send_dump_to_remote_node(FromPid, Dump, State); -handle_call({insert, Rec}, _From, State = #{tab := Tab}) -> - ets:insert(Tab, Rec), - {reply, ok, State}. +handle_call({insert, Rec}, From, State) -> + handle_insert(Rec, From, State); +handle_call({delete, Keys}, From, State) -> + handle_delete(Keys, From, State). handle_cast(_Msg, State) -> {noreply, State}. @@ -300,3 +277,36 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> reply_updated(Pid, Mon) -> %% We really don't wanna block this process erlang:send(Pid, {updated, Mon}, [noconnect, nosuspend]). + +send_to_remote(RemotePid, Msg) -> + erlang:send(RemotePid, Msg, [noconnect, nosuspend]). + +handle_insert(Rec, _From = {FromPid, _}, State = #{tab := Tab, other_servers := Servers}) -> + ets:insert(Tab, Rec), + %% Insert to other nodes and block till written + Monitors = insert_to_remote_nodes(Servers, Rec, FromPid), + {reply, {ok, Monitors}, State}. + +insert_to_remote_nodes([{RemotePid, ProxyPid} | Servers], Rec, FromPid) -> + Mon = erlang:monitor(process, ProxyPid), + %% Reply would be routed directly to FromPid + Msg = {insert_from_remote_node, Mon, FromPid, Rec}, + send_to_remote(RemotePid, Msg), + [Mon|insert_to_remote_nodes(Servers, Rec, FromPid)]; +insert_to_remote_nodes([], _Rec, _FromPid) -> + []. + +handle_delete(Keys, _From = {FromPid, _}, State = #{tab := Tab, other_servers := Servers}) -> + ets_delete_keys(Tab, Keys), + %% Insert to other nodes and block till written + Monitors = delete_from_remote_nodes(Servers, Keys, FromPid), + {reply, {ok, Monitors}, State}. + +delete_from_remote_nodes([{RemotePid, ProxyPid} | Servers], Keys, FromPid) -> + Mon = erlang:monitor(process, ProxyPid), + %% Reply would be routed directly to FromPid + Msg = {delete_from_remote_node, Mon, FromPid, Keys}, + send_to_remote(RemotePid, Msg), + [Mon|delete_from_remote_nodes(Servers, Keys, FromPid)]; +delete_from_remote_nodes([], _Keys, _FromPid) -> + []. From 813d1066189e2e1672dbdc1c4f4ea0def7ef1bad Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 4 Mar 2022 11:20:32 +0100 Subject: [PATCH 17/64] Remove kiss_pt --- src/kiss.erl | 15 ++++++--------- src/kiss_pt.erl | 36 ------------------------------------ 2 files changed, 6 insertions(+), 45 deletions(-) delete mode 100644 src/kiss_pt.erl diff --git a/src/kiss.erl b/src/kiss.erl index b7c02f92..aad95fa1 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -117,8 +117,8 @@ wait_for_updated([Mon | Monitors]) -> wait_for_updated([]) -> ok. -other_servers(Tab) -> - kiss_pt:get(Tab). +other_servers(Server) -> + gen_server:call(Server, other_servers). other_nodes(Tab) -> lists:usort(pids_to_nodes(servers_to_pids(other_servers(Tab)))). @@ -126,9 +126,10 @@ other_nodes(Tab) -> init([Tab, Opts]) -> ets:new(Tab, [ordered_set, named_table, public, {read_concurrency, true}]), - kiss_pt:put(Tab, []), {ok, #{tab => Tab, other_servers => [], opts => Opts}}. +handle_call(other_servers, _From, State = #{other_servers := Servers}) -> + {reply, Servers, State}; handle_call({join, RemotePid}, _From, State) -> handle_join(RemotePid, State); handle_call({remote_add_node_to_schema, ServerPid, OtherPids, ReturnDump}, _From, State) -> @@ -154,8 +155,7 @@ handle_info({delete_from_remote_node, Mon, Pid, Keys}, State = #{tab := Tab}) -> reply_updated(Pid, Mon), {noreply, State}. -terminate(_Reason, _State = #{tab := Tab}) -> - kiss_pt:put(Tab, []), +terminate(_Reason, _State) -> ok. code_change(_OldVsn, State, _Extra) -> @@ -191,7 +191,6 @@ handle_join3(RemotePid, Dump, OtherPids, KnownPids, [remote_add_node_to_schema(Pid, Self, NewPids, false) || Pid <- KnownPids], Servers2 = add_servers(NewPids, Servers), %% Ask our node to replicate data there before applying the dump - kiss_pt:put(Tab, Servers2), OurDump = dump(Tab), %% A race condition is possible: when the remote node inserts a deleted record %% Send to all nodes from that partition @@ -207,7 +206,6 @@ handle_join3(RemotePid, Dump, OtherPids, KnownPids, handle_remote_add_node_to_schema(RemotePid, OtherPids, ReturnDump, State = #{tab := Tab, other_servers := Servers}) -> Servers2 = add_servers([RemotePid | OtherPids], Servers), - kiss_pt:put(Tab, Servers2), KnownPids = servers_to_pids(Servers), Dump = case ReturnDump of true -> dump(Tab); false -> not_requested end, {reply, {ok, Dump, KnownPids}, State#{other_servers => Servers2}}. @@ -216,11 +214,10 @@ handle_send_dump_to_remote_node(_FromPid, Dump, State = #{tab := Tab}) -> ets:insert(Tab, Dump), {reply, ok, State}. -handle_down(ProxyPid, State = #{tab := Tab, other_servers := Servers}) -> +handle_down(ProxyPid, State = #{other_servers := Servers}) -> case lists:keytake(ProxyPid, 2, Servers) of {value, {RemotePid, _}, Servers2} -> %% Down from a proxy - kiss_pt:put(Tab, Servers2), call_user_handle_down(RemotePid, State), {noreply, State#{other_servers => Servers2}}; false -> diff --git a/src/kiss_pt.erl b/src/kiss_pt.erl deleted file mode 100644 index 34af5cb1..00000000 --- a/src/kiss_pt.erl +++ /dev/null @@ -1,36 +0,0 @@ --module(kiss_pt). --export([put/2, get/1]). - --compile({no_auto_import,[get/1]}). - -%% Avoids GC -put(Tab, Data) -> - try get(Tab) of - Data -> - ok; - _ -> - just_put(Tab, Data) - catch _:_ -> - just_put(Tab, Data) - end. - -just_put(Tab, Data) -> - NextKey = next_key(Tab), - persistent_term:put(NextKey, Data), - persistent_term:put(Tab, NextKey). - -get(Tab) -> - Key = persistent_term:get(Tab), - persistent_term:get(Key). - -next_key(Tab) -> - try - Key = persistent_term:get(Tab), - N = list_to_integer(get_suffix(atom_to_list(Key), atom_to_list(Tab) ++ "_")) + 1, - list_to_atom(atom_to_list(Tab) ++ "_" ++ integer_to_list(N)) - catch _:_ -> - list_to_atom(atom_to_list(Tab) ++ "_1") - end. - -get_suffix(Str, Prefix) -> - lists:sublist(Str, length(Prefix) + 1, length(Str) - length(Prefix)). From 519dfaa4a893f591d20e9fe117bbb16f7a3dad86 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 4 Mar 2022 11:59:44 +0100 Subject: [PATCH 18/64] Properly pause to avoid race conditions --- src/kiss.erl | 74 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index aad95fa1..b59a8acb 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -84,6 +84,12 @@ remote_add_node_to_schema(RemotePid, ServerPid, OtherPids, ReturnDump) -> other_pids => OtherPids, other_nodes => pids_to_nodes(OtherPids)}, kiss_long:run_safely(Info, F). +get_other_pids(RemotePid) -> + F = fun() -> gen_server:call(RemotePid, get_other_pids, infinity) end, + Info = #{task => get_other_pids, + remote_pid => RemotePid, remote_node => node(RemotePid)}, + kiss_long:run_safely(Info, F). + send_dump_to_remote_node(_RemotePid, _FromPid, []) -> skipped; send_dump_to_remote_node(RemotePid, FromPid, OurDump) -> @@ -123,19 +129,41 @@ other_servers(Server) -> other_nodes(Tab) -> lists:usort(pids_to_nodes(servers_to_pids(other_servers(Tab)))). +pause(RemotePid) -> + F = fun() -> gen_server:call(RemotePid, pause, infinity) end, + Info = #{task => pause, + remote_pid => RemotePid, remote_node => node(RemotePid)}, + kiss_long:run_safely(Info, F). + +unpause(RemotePid) -> + F = fun() -> gen_server:call(RemotePid, unpause, infinity) end, + Info = #{task => unpause, + remote_pid => RemotePid, remote_node => node(RemotePid)}, + kiss_long:run_safely(Info, F). + init([Tab, Opts]) -> ets:new(Tab, [ordered_set, named_table, public, {read_concurrency, true}]), - {ok, #{tab => Tab, other_servers => [], opts => Opts}}. + {ok, #{tab => Tab, other_servers => [], opts => Opts, backlog => [], + paused => false, pause_monitor => undefined}}. handle_call(other_servers, _From, State = #{other_servers := Servers}) -> {reply, Servers, State}; +handle_call(get_other_pids, _From, State = #{other_servers := Servers}) -> + {reply, {ok, servers_to_pids(Servers)}, State}; handle_call({join, RemotePid}, _From, State) -> handle_join(RemotePid, State); handle_call({remote_add_node_to_schema, ServerPid, OtherPids, ReturnDump}, _From, State) -> handle_remote_add_node_to_schema(ServerPid, OtherPids, ReturnDump, State); handle_call({send_dump_to_remote_node, FromPid, Dump}, _From, State) -> handle_send_dump_to_remote_node(FromPid, Dump, State); +handle_call(pause, _From = {FromPid, _}, State) -> + Mon = erlang:monitor(process, FromPid), + {reply, ok, State#{pause => true, pause_monitor => Mon}}; +handle_call(unpause, _From, State) -> + handle_unpause(State); +handle_call(Msg, From, State = #{paused := true, backlog := Backlog}) -> + {noreply, State#{backlog => [{Msg, From} | Backlog]}}; handle_call({insert, Rec}, From, State) -> handle_insert(Rec, From, State); handle_call({delete, Keys}, From, State) -> @@ -144,8 +172,8 @@ handle_call({delete, Keys}, From, State) -> handle_cast(_Msg, State) -> {noreply, State}. -handle_info({'DOWN', _Mon, process, Pid, _Reason}, State) -> - handle_down(Pid, State); +handle_info({'DOWN', Mon, process, Pid, _Reason}, State) -> + handle_down(Mon, Pid, State); handle_info({insert_from_remote_node, Mon, Pid, Rec}, State = #{tab := Tab}) -> ets:insert(Tab, Rec), reply_updated(Pid, Mon), @@ -173,16 +201,31 @@ handle_join(RemotePid, State = #{other_servers := Servers}) when is_pid(RemotePi handle_join2(RemotePid, State = #{other_servers := Servers}) -> KnownPids = servers_to_pids(Servers), + case get_other_pids(RemotePid) of + {ok, OtherPids} -> + AllPids = KnownPids ++ OtherPids, + [pause(Pid) || Pid <- AllPids], + try + handle_join3(RemotePid, OtherPids, KnownPids, State) + after + [unpause(Pid) || Pid <- AllPids] + end; + Other -> + ?LOG_ERROR(#{what => get_other_pids_failed, reason => Other}), + {reply, {error, get_other_pids_failed}, State} + end. + +handle_join3(RemotePid, OtherPids, KnownPids, State) -> %% Remote gen_server calls here are "safe" case remote_add_node_to_schema(RemotePid, self(), KnownPids, true) of {ok, Dump, OtherPids} -> - handle_join3(RemotePid, Dump, OtherPids, KnownPids, State); + handle_join4(RemotePid, Dump, OtherPids, KnownPids, State); Other -> ?LOG_ERROR(#{what => remote_add_node_to_schema, reason => Other}), {reply, {error, remote_add_node_to_schema_failed}, State} end. -handle_join3(RemotePid, Dump, OtherPids, KnownPids, +handle_join4(RemotePid, Dump, OtherPids, KnownPids, State = #{tab := Tab, other_servers := Servers}) -> Self = self(), NewPids = [RemotePid | OtherPids], @@ -214,7 +257,10 @@ handle_send_dump_to_remote_node(_FromPid, Dump, State = #{tab := Tab}) -> ets:insert(Tab, Dump), {reply, ok, State}. -handle_down(ProxyPid, State = #{other_servers := Servers}) -> +handle_down(Mon, PausedByPid, State = #{pause_monitor := Mon}) -> + ?LOG_ERROR(#{what => pause_owner_crashed, state => State, paused_by_pid => PausedByPid}), + handle_unpause(State); +handle_down(_Mon, ProxyPid, State = #{other_servers := Servers}) -> case lists:keytake(ProxyPid, 2, Servers) of {value, {RemotePid, _}, Servers2} -> %% Down from a proxy @@ -307,3 +353,19 @@ delete_from_remote_nodes([{RemotePid, ProxyPid} | Servers], Keys, FromPid) -> [Mon|delete_from_remote_nodes(Servers, Keys, FromPid)]; delete_from_remote_nodes([], _Keys, _FromPid) -> []. + +apply_backlog([{Msg, From}|Backlog], State) -> + {reply, Reply, State2} = handle_call(Msg, From, State), + gen_server:reply(From, Reply), + apply_backlog(Backlog, State2); +apply_backlog([], State) -> + State. + +%% Theoretically we can support mupltiple pauses (but no need for now because +%% we pause in the global locked function) +handle_unpause(State = #{pause := false}) -> + {reply, {error, already_unpaused}, State}; +handle_unpause(State = #{backlog := Backlog, pause_monitor := Mon}) -> + erlang:demonitor(Mon, [flush]), + State2 = State#{pause => false, backlog := [], pause_monitor => undefined}, + {reply, ok, apply_backlog(Backlog, State2)}. From 90dd5e0a33e4cfe79278c908c9e516deff2d6c1a Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 4 Mar 2022 16:24:10 +0100 Subject: [PATCH 19/64] Move joining logic into a separate file --- src/kiss.erl | 173 ++++++++++------------------------------- src/kiss_discovery.erl | 5 +- src/kiss_join.erl | 60 ++++++++++++++ test/kiss_SUITE.erl | 48 ++++++++---- 4 files changed, 136 insertions(+), 150 deletions(-) create mode 100644 src/kiss_join.erl diff --git a/src/kiss.erl b/src/kiss.erl index b59a8acb..91978eff 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -13,7 +13,10 @@ -module(kiss). -behaviour(gen_server). --export([start/2, stop/1, dump/1, insert/2, delete/2, delete_many/2, join/3, other_nodes/1]). +-export([start/2, stop/1, insert/2, delete/2, delete_many/2]). +-export([dump/1, remote_dump/1, send_dump_to_remote_node/3]). +-export([other_nodes/1, other_pids/1]). +-export([pause/1, unpause/1, sync/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). @@ -37,63 +40,14 @@ stop(Tab) -> dump(Tab) -> ets:tab2list(Tab). -%% Adds a node to a cluster. -%% Writes from other nodes would wait for join completion. -%% LockKey should be the same on all nodes. -join(LockKey, Tab, RemotePid) when is_pid(RemotePid) -> - Servers = other_servers(Tab), - case lists:keymember(RemotePid, 1, Servers) of - true -> - {error, already_joined}; - false -> - Start = os:timestamp(), - F = fun() -> join_loop(LockKey, Tab, RemotePid, Start) end, - kiss_long:run(#{task => join, table => Tab, - remote_pid => RemotePid, - remote_node => node(RemotePid)}, F) - end. - -join_loop(LockKey, Tab, RemotePid, Start) -> - %% Only one join at a time: - %% - for performance reasons, we don't want to cause too much load for active nodes - %% - to avoid deadlocks, because joining does gen_server calls - F = fun() -> - Diff = timer:now_diff(os:timestamp(), Start) div 1000, - %% Getting the lock could take really long time in case nodes are - %% overloaded or joining is already in progress on another node - ?LOG_INFO(#{what => join_got_lock, table => Tab, after_time_ms => Diff}), - gen_server:call(Tab, {join, RemotePid}, infinity) - end, - LockRequest = {LockKey, self()}, - %% Just lock all nodes, no magic here :) - Nodes = [node() | nodes()], - Retries = 1, - case global:trans(LockRequest, F, Nodes, Retries) of - aborted -> - ?LOG_ERROR(#{what => join_retry, reason => lock_aborted}), - join_loop(LockKey, Tab, RemotePid, Start); - Result -> - Result - end. - -remote_add_node_to_schema(RemotePid, ServerPid, OtherPids, ReturnDump) -> - Msg = {remote_add_node_to_schema, ServerPid, OtherPids, ReturnDump}, - F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, - Info = #{task => remote_add_node_to_schema, return_dump => ReturnDump, - remote_pid => RemotePid, remote_node => node(RemotePid), - other_pids => OtherPids, other_nodes => pids_to_nodes(OtherPids)}, - kiss_long:run_safely(Info, F). - -get_other_pids(RemotePid) -> - F = fun() -> gen_server:call(RemotePid, get_other_pids, infinity) end, - Info = #{task => get_other_pids, +remote_dump(RemotePid) -> + F = fun() -> gen_server:call(RemotePid, remote_dump, infinity) end, + Info = #{task => remote_dump, remote_pid => RemotePid, remote_node => node(RemotePid)}, kiss_long:run_safely(Info, F). -send_dump_to_remote_node(_RemotePid, _FromPid, []) -> - skipped; -send_dump_to_remote_node(RemotePid, FromPid, OurDump) -> - Msg = {send_dump_to_remote_node, FromPid, OurDump}, +send_dump_to_remote_node(RemotePid, NewPids, OurDump) -> + Msg = {send_dump_to_remote_node, NewPids, OurDump}, F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, Info = #{task => send_dump_to_remote_node, remote_pid => RemotePid, count => length(OurDump)}, @@ -126,8 +80,11 @@ wait_for_updated([]) -> other_servers(Server) -> gen_server:call(Server, other_servers). -other_nodes(Tab) -> - lists:usort(pids_to_nodes(servers_to_pids(other_servers(Tab)))). +other_nodes(Server) -> + lists:usort(pids_to_nodes(servers_to_pids(other_servers(Server)))). + +other_pids(Server) -> + servers_to_pids(other_servers(Server)). pause(RemotePid) -> F = fun() -> gen_server:call(RemotePid, pause, infinity) end, @@ -141,22 +98,35 @@ unpause(RemotePid) -> remote_pid => RemotePid, remote_node => node(RemotePid)}, kiss_long:run_safely(Info, F). +sync(RemotePid) -> + F = fun() -> gen_server:call(RemotePid, sync, infinity) end, + Info = #{task => sync, + remote_pid => RemotePid, remote_node => node(RemotePid)}, + kiss_long:run_safely(Info, F). + +ping(RemotePid) -> + F = fun() -> gen_server:call(RemotePid, ping, infinity) end, + Info = #{task => ping, + remote_pid => RemotePid, remote_node => node(RemotePid)}, + kiss_long:run_safely(Info, F). + init([Tab, Opts]) -> ets:new(Tab, [ordered_set, named_table, public, {read_concurrency, true}]), {ok, #{tab => Tab, other_servers => [], opts => Opts, backlog => [], paused => false, pause_monitor => undefined}}. +handle_call(remote_dump, _From, State = #{tab := Tab}) -> + {reply, {ok, dump(Tab)}, State}; handle_call(other_servers, _From, State = #{other_servers := Servers}) -> {reply, Servers, State}; -handle_call(get_other_pids, _From, State = #{other_servers := Servers}) -> - {reply, {ok, servers_to_pids(Servers)}, State}; -handle_call({join, RemotePid}, _From, State) -> - handle_join(RemotePid, State); -handle_call({remote_add_node_to_schema, ServerPid, OtherPids, ReturnDump}, _From, State) -> - handle_remote_add_node_to_schema(ServerPid, OtherPids, ReturnDump, State); -handle_call({send_dump_to_remote_node, FromPid, Dump}, _From, State) -> - handle_send_dump_to_remote_node(FromPid, Dump, State); +handle_call(sync, _From, State) -> + handle_sync(State), + {reply, ok, State}; +handle_call(ping, _From, State) -> + {reply, ping, State}; +handle_call({send_dump_to_remote_node, NewPids, Dump}, _From, State) -> + handle_send_dump_to_remote_node(NewPids, Dump, State); handle_call(pause, _From = {FromPid, _}, State) -> Mon = erlang:monitor(process, FromPid), {reply, ok, State#{pause => true, pause_monitor => Mon}}; @@ -189,74 +159,11 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. - -handle_join(RemotePid, State = #{other_servers := Servers}) when is_pid(RemotePid) -> - case has_remote_pid(RemotePid, Servers) of - true -> - %% Already added - {reply, ok, State}; - false -> - handle_join2(RemotePid, State) - end. - -handle_join2(RemotePid, State = #{other_servers := Servers}) -> - KnownPids = servers_to_pids(Servers), - case get_other_pids(RemotePid) of - {ok, OtherPids} -> - AllPids = KnownPids ++ OtherPids, - [pause(Pid) || Pid <- AllPids], - try - handle_join3(RemotePid, OtherPids, KnownPids, State) - after - [unpause(Pid) || Pid <- AllPids] - end; - Other -> - ?LOG_ERROR(#{what => get_other_pids_failed, reason => Other}), - {reply, {error, get_other_pids_failed}, State} - end. - -handle_join3(RemotePid, OtherPids, KnownPids, State) -> - %% Remote gen_server calls here are "safe" - case remote_add_node_to_schema(RemotePid, self(), KnownPids, true) of - {ok, Dump, OtherPids} -> - handle_join4(RemotePid, Dump, OtherPids, KnownPids, State); - Other -> - ?LOG_ERROR(#{what => remote_add_node_to_schema, reason => Other}), - {reply, {error, remote_add_node_to_schema_failed}, State} - end. - -handle_join4(RemotePid, Dump, OtherPids, KnownPids, - State = #{tab := Tab, other_servers := Servers}) -> - Self = self(), - NewPids = [RemotePid | OtherPids], - %% Let all nodes to know each other - [remote_add_node_to_schema(Pid, Self, KnownPids, false) || Pid <- OtherPids], - [remote_add_node_to_schema(Pid, Self, NewPids, false) || Pid <- KnownPids], - Servers2 = add_servers(NewPids, Servers), - %% Ask our node to replicate data there before applying the dump - OurDump = dump(Tab), - %% A race condition is possible: when the remote node inserts a deleted record - %% Send to all nodes from that partition - [send_dump_to_remote_node(Pid, Self, OurDump) || Pid <- NewPids], - %% Apply to our nodes - [send_dump_to_remote_node(Pid, Self, Dump) || Pid <- KnownPids], +handle_send_dump_to_remote_node(NewPids, Dump, State = #{tab := Tab, other_servers := Servers}) -> ets:insert(Tab, Dump), - %% Add ourself into remote schema - %% Add remote nodes into our schema - %% Copy from our node / Copy into our node + Servers2 = add_servers(NewPids, Servers), {reply, ok, State#{other_servers => Servers2}}. -handle_remote_add_node_to_schema(RemotePid, OtherPids, ReturnDump, - State = #{tab := Tab, other_servers := Servers}) -> - Servers2 = add_servers([RemotePid | OtherPids], Servers), - KnownPids = servers_to_pids(Servers), - Dump = case ReturnDump of true -> dump(Tab); false -> not_requested end, - {reply, {ok, Dump, KnownPids}, State#{other_servers => Servers2}}. - -handle_send_dump_to_remote_node(_FromPid, Dump, State = #{tab := Tab}) -> - ets:insert(Tab, Dump), - {reply, ok, State}. - handle_down(Mon, PausedByPid, State = #{pause_monitor := Mon}) -> ?LOG_ERROR(#{what => pause_owner_crashed, state => State, paused_by_pid => PausedByPid}), handle_unpause(State); @@ -368,4 +275,8 @@ handle_unpause(State = #{pause := false}) -> handle_unpause(State = #{backlog := Backlog, pause_monitor := Mon}) -> erlang:demonitor(Mon, [flush]), State2 = State#{pause => false, backlog := [], pause_monitor => undefined}, - {reply, ok, apply_backlog(Backlog, State2)}. + {reply, ok, apply_backlog(lists:reverse(Backlog), State2)}. + +handle_sync(#{other_servers := Servers}) -> + [ping(Pid) || Pid <- servers_to_pids(Servers)], + ok. diff --git a/src/kiss_discovery.erl b/src/kiss_discovery.erl index bf4e2549..e0765bf4 100644 --- a/src/kiss_discovery.erl +++ b/src/kiss_discovery.erl @@ -90,10 +90,11 @@ cancel_old_timer(_State) -> ok. do_join(Tab, Node) -> + LocalPid = whereis(Tab), %% That would trigger autoconnect for the first time case rpc:call(Node, erlang, whereis, [Tab]) of - Pid when is_pid(Pid) -> - Result = kiss:join(kiss_discovery, Tab, Pid), + Pid when is_pid(Pid), is_pid(LocalPid) -> + Result = kiss_join:join(kiss_discovery, #{table => Tab}, LocalPid, Pid), #{what => join_result, result => Result, node => Node, table => Tab}; Other -> #{what => pid_not_found, reason => Other, node => Node, table => Tab} diff --git a/src/kiss_join.erl b/src/kiss_join.erl new file mode 100644 index 00000000..7bbb31aa --- /dev/null +++ b/src/kiss_join.erl @@ -0,0 +1,60 @@ +-module(kiss_join). +-export([join/4]). +-include_lib("kernel/include/logger.hrl"). + +%% Adds a node to a cluster. +%% Writes from other nodes would wait for join completion. +%% LockKey should be the same on all nodes. +join(LockKey, Info, LocalPid, RemotePid) when is_pid(LocalPid), is_pid(RemotePid) -> + Info2 = Info#{local_pid => LocalPid, remote_pid => RemotePid, remote_node => node(RemotePid)}, + OtherPids = kiss:other_pids(LocalPid), + case lists:member(RemotePid, OtherPids) of + true -> + {error, already_joined}; + false -> + Start = os:timestamp(), + F = fun() -> join_loop(LockKey, Info2, LocalPid, RemotePid, Start) end, + kiss_long:run(Info2#{task => join}, F) + end. + +join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> + %% Only one join at a time: + %% - for performance reasons, we don't want to cause too much load for active nodes + %% - to avoid deadlocks, because joining does gen_server calls + F = fun() -> + Diff = timer:now_diff(os:timestamp(), Start) div 1000, + %% Getting the lock could take really long time in case nodes are + %% overloaded or joining is already in progress on another node + ?LOG_INFO(Info#{what => join_got_lock, after_time_ms => Diff}), + join2(Info, LocalPid, RemotePid) + end, + LockRequest = {LockKey, self()}, + %% Just lock all nodes, no magic here :) + Nodes = [node() | nodes()], + Retries = 1, + case global:trans(LockRequest, F, Nodes, Retries) of + aborted -> + ?LOG_ERROR(Info#{what => join_retry, reason => lock_aborted}), + join_loop(LockKey, Info, LocalPid, RemotePid, Start); + Result -> + Result + end. + +join2(_Info, LocalPid, RemotePid) -> + LocalOtherPids = kiss:other_pids(LocalPid), + RemoteOtherPids = kiss:other_pids(RemotePid), + LocPids = [LocalPid | LocalOtherPids], + RemPids = [RemotePid | RemoteOtherPids], + AllPids = LocPids ++ RemPids, + [kiss:pause(Pid) || Pid <- AllPids], + try + kiss:sync(LocalPid), + kiss:sync(RemotePid), + {ok, LocalDump} = kiss:remote_dump(LocalPid), + {ok, RemoteDump} = kiss:remote_dump(RemotePid), + [kiss:send_dump_to_remote_node(Pid, LocPids, LocalDump) || Pid <- RemPids], + [kiss:send_dump_to_remote_node(Pid, RemPids, RemoteDump) || Pid <- LocPids], + ok + after + [kiss:unpause(Pid) || Pid <- AllPids] + end. diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index 0b4558cc..5702f23f 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -1,9 +1,10 @@ -module(kiss_SUITE). -include_lib("common_test/include/ct.hrl"). --compile([export_all]). +-compile([export_all, nowarn_export_all]). -all() -> [test_multinode, test_multinode_auto_discovery, test_locally, +all() -> [test_multinode, node_list_is_correct, + test_multinode_auto_discovery, test_locally, handle_down_is_called]. init_per_suite(Config) -> @@ -28,19 +29,19 @@ test_multinode(Config) -> Node1 = node(), [Node2, Node3, Node4] = proplists:get_value(nodes, Config), Tab = tab1, - {ok, _Pid1} = start(Node1, Tab), + {ok, Pid1} = start(Node1, Tab), {ok, Pid2} = start(Node2, Tab), {ok, Pid3} = start(Node3, Tab), {ok, Pid4} = start(Node4, Tab), - join(Node1, Pid3, Tab), - join(Node2, Pid4, Tab), + ok = join(Node1, Tab, Pid3, Pid1), + ok = join(Node2, Tab, Pid4, Pid2), insert(Node1, Tab, {a}), insert(Node2, Tab, {b}), insert(Node3, Tab, {c}), insert(Node4, Tab, {d}), [{a}, {c}] = dump(Node1, Tab), [{b}, {d}] = dump(Node2, Tab), - join(Node1, Pid2, Tab), + ok = join(Node1, Tab, Pid2, Pid1), [{a}, {b}, {c}, {d}] = dump(Node1, Tab), [{a}, {b}, {c}, {d}] = dump(Node2, Tab), insert(Node1, Tab, {f}), @@ -52,10 +53,6 @@ test_multinode(Config) -> X = dump(Node4, Tab) end, Same([{a}, {b}, {c}, {d}, {e}, {f}]), - [Node2, Node3, Node4] = other_nodes(Node1, Tab), - [Node1, Node3, Node4] = other_nodes(Node2, Tab), - [Node1, Node2, Node4] = other_nodes(Node3, Tab), - [Node1, Node2, Node3] = other_nodes(Node4, Tab), delete(Node1, Tab, e), Same([{a}, {b}, {c}, {d}, {f}]), delete(Node4, Tab, a), @@ -67,12 +64,29 @@ test_multinode(Config) -> Same([{b}, {c}, {d}, {f}, {m}, {y}]), ok. -test_multinode_auto_discovery(Config) -> +node_list_is_correct(Config) -> Node1 = node(), [Node2, Node3, Node4] = proplists:get_value(nodes, Config), + Tab = tab3, + {ok, Pid1} = start(Node1, Tab), + {ok, Pid2} = start(Node2, Tab), + {ok, Pid3} = start(Node3, Tab), + {ok, Pid4} = start(Node4, Tab), + ok = join(Node1, Tab, Pid3, Pid1), + ok = join(Node2, Tab, Pid4, Pid2), + ok = join(Node1, Tab, Pid2, Pid1), + [Node2, Node3, Node4] = other_nodes(Node1, Tab), + [Node1, Node3, Node4] = other_nodes(Node2, Tab), + [Node1, Node2, Node4] = other_nodes(Node3, Tab), + [Node1, Node2, Node3] = other_nodes(Node4, Tab), + ok. + +test_multinode_auto_discovery(Config) -> + Node1 = node(), + [Node2, _Node3, _Node4] = proplists:get_value(nodes, Config), Tab = tab2, {ok, _Pid1} = start(Node1, Tab), - {ok, Pid2} = start(Node2, Tab), + {ok, _Pid2} = start(Node2, Tab), Dir = proplists:get_value(priv_dir, Config), ct:pal("Dir ~p", [Dir]), FileName = filename:join(Dir, "disco.txt"), @@ -84,9 +98,9 @@ test_multinode_auto_discovery(Config) -> ok. test_locally(_Config) -> - {ok, _Pid1} = kiss:start(t1, #{}), + {ok, Pid1} = kiss:start(t1, #{}), {ok, Pid2} = kiss:start(t2, #{}), - kiss:join(lock1, t1, Pid2), + ok = kiss_join:join(lock1, #{table => [t1, t2]}, Pid1, Pid2), kiss:insert(t1, {1}), kiss:insert(t1, {1}), kiss:insert(t2, {2}), @@ -100,7 +114,7 @@ handle_down_is_called(_Config) -> end, {ok, Pid1} = kiss:start(d1, #{handle_down => DownFn}), {ok, Pid2} = kiss:start(d2, #{}), - kiss:join(lock1, d1, Pid2), + ok = kiss_join:join(lock1, #{table => [d1, d2]}, Pid1, Pid2), exit(Pid2, oops), receive down_called -> ok @@ -125,8 +139,8 @@ dump(Node, Tab) -> other_nodes(Node, Tab) -> rpc(Node, kiss, other_nodes, [Tab]). -join(Node1, Node2, Tab) -> - rpc(Node1, kiss, join, [lock1, Tab, Node2]). +join(Node1, Tab, Pid1, Pid2) -> + rpc(Node1, kiss_join, join, [lock1, #{table => Tab}, Pid1, Pid2]). rpc(Node, M, F, Args) -> case rpc:call(Node, M, F, Args) of From 0f668b812c6de1f3a67ebc5edc26a1db59ead9bf Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 4 Mar 2022 16:29:22 +0100 Subject: [PATCH 20/64] Dedup code with short_call --- src/kiss.erl | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 91978eff..b7998d09 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -41,10 +41,7 @@ dump(Tab) -> ets:tab2list(Tab). remote_dump(RemotePid) -> - F = fun() -> gen_server:call(RemotePid, remote_dump, infinity) end, - Info = #{task => remote_dump, - remote_pid => RemotePid, remote_node => node(RemotePid)}, - kiss_long:run_safely(Info, F). + short_call(RemotePid, remote_dump). send_dump_to_remote_node(RemotePid, NewPids, OurDump) -> Msg = {send_dump_to_remote_node, NewPids, OurDump}, @@ -81,32 +78,26 @@ other_servers(Server) -> gen_server:call(Server, other_servers). other_nodes(Server) -> - lists:usort(pids_to_nodes(servers_to_pids(other_servers(Server)))). + lists:usort(pids_to_nodes(other_pids(Server))). other_pids(Server) -> servers_to_pids(other_servers(Server)). pause(RemotePid) -> - F = fun() -> gen_server:call(RemotePid, pause, infinity) end, - Info = #{task => pause, - remote_pid => RemotePid, remote_node => node(RemotePid)}, - kiss_long:run_safely(Info, F). + short_call(RemotePid, pause). unpause(RemotePid) -> - F = fun() -> gen_server:call(RemotePid, unpause, infinity) end, - Info = #{task => unpause, - remote_pid => RemotePid, remote_node => node(RemotePid)}, - kiss_long:run_safely(Info, F). + short_call(RemotePid, unpause). sync(RemotePid) -> - F = fun() -> gen_server:call(RemotePid, sync, infinity) end, - Info = #{task => sync, - remote_pid => RemotePid, remote_node => node(RemotePid)}, - kiss_long:run_safely(Info, F). + short_call(RemotePid, sync). ping(RemotePid) -> - F = fun() -> gen_server:call(RemotePid, ping, infinity) end, - Info = #{task => ping, + short_call(RemotePid, ping). + +short_call(RemotePid, Msg) -> + F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, + Info = #{task => Msg, remote_pid => RemotePid, remote_node => node(RemotePid)}, kiss_long:run_safely(Info, F). @@ -242,7 +233,7 @@ insert_to_remote_nodes([{RemotePid, ProxyPid} | Servers], Rec, FromPid) -> %% Reply would be routed directly to FromPid Msg = {insert_from_remote_node, Mon, FromPid, Rec}, send_to_remote(RemotePid, Msg), - [Mon|insert_to_remote_nodes(Servers, Rec, FromPid)]; + [Mon | insert_to_remote_nodes(Servers, Rec, FromPid)]; insert_to_remote_nodes([], _Rec, _FromPid) -> []. @@ -257,7 +248,7 @@ delete_from_remote_nodes([{RemotePid, ProxyPid} | Servers], Keys, FromPid) -> %% Reply would be routed directly to FromPid Msg = {delete_from_remote_node, Mon, FromPid, Keys}, send_to_remote(RemotePid, Msg), - [Mon|delete_from_remote_nodes(Servers, Keys, FromPid)]; + [Mon | delete_from_remote_nodes(Servers, Keys, FromPid)]; delete_from_remote_nodes([], _Keys, _FromPid) -> []. From 8dafa78807e99670055f9b7057b98ab59cf3e767 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 4 Mar 2022 16:42:50 +0100 Subject: [PATCH 21/64] Merge insert_to_remote_nodes and delete_from_remote_nodes --- src/kiss.erl | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index b7998d09..2c6a3935 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -107,6 +107,11 @@ init([Tab, Opts]) -> {ok, #{tab => Tab, other_servers => [], opts => Opts, backlog => [], paused => false, pause_monitor => undefined}}. + +handle_call({insert, Rec}, From, State = #{paused := false}) -> + handle_insert(Rec, From, State); +handle_call({delete, Keys}, From, State = #{paused := false}) -> + handle_delete(Keys, From, State); handle_call(remote_dump, _From, State = #{tab := Tab}) -> {reply, {ok, dump(Tab)}, State}; handle_call(other_servers, _From, State = #{other_servers := Servers}) -> @@ -124,25 +129,21 @@ handle_call(pause, _From = {FromPid, _}, State) -> handle_call(unpause, _From, State) -> handle_unpause(State); handle_call(Msg, From, State = #{paused := true, backlog := Backlog}) -> - {noreply, State#{backlog => [{Msg, From} | Backlog]}}; -handle_call({insert, Rec}, From, State) -> - handle_insert(Rec, From, State); -handle_call({delete, Keys}, From, State) -> - handle_delete(Keys, From, State). + {noreply, State#{backlog => [{Msg, From} | Backlog]}}. handle_cast(_Msg, State) -> {noreply, State}. -handle_info({'DOWN', Mon, process, Pid, _Reason}, State) -> - handle_down(Mon, Pid, State); -handle_info({insert_from_remote_node, Mon, Pid, Rec}, State = #{tab := Tab}) -> +handle_info({remote_insert, Mon, Pid, Rec}, State = #{tab := Tab}) -> ets:insert(Tab, Rec), reply_updated(Pid, Mon), {noreply, State}; -handle_info({delete_from_remote_node, Mon, Pid, Keys}, State = #{tab := Tab}) -> +handle_info({remote_delete, Mon, Pid, Keys}, State = #{tab := Tab}) -> ets_delete_keys(Tab, Keys), reply_updated(Pid, Mon), - {noreply, State}. + {noreply, State}; +handle_info({'DOWN', Mon, process, Pid, _Reason}, State) -> + handle_down(Mon, Pid, State). terminate(_Reason, _State) -> ok. @@ -225,31 +226,22 @@ send_to_remote(RemotePid, Msg) -> handle_insert(Rec, _From = {FromPid, _}, State = #{tab := Tab, other_servers := Servers}) -> ets:insert(Tab, Rec), %% Insert to other nodes and block till written - Monitors = insert_to_remote_nodes(Servers, Rec, FromPid), + Monitors = replicate(Servers, remote_insert, Rec, FromPid), {reply, {ok, Monitors}, State}. -insert_to_remote_nodes([{RemotePid, ProxyPid} | Servers], Rec, FromPid) -> - Mon = erlang:monitor(process, ProxyPid), - %% Reply would be routed directly to FromPid - Msg = {insert_from_remote_node, Mon, FromPid, Rec}, - send_to_remote(RemotePid, Msg), - [Mon | insert_to_remote_nodes(Servers, Rec, FromPid)]; -insert_to_remote_nodes([], _Rec, _FromPid) -> - []. - handle_delete(Keys, _From = {FromPid, _}, State = #{tab := Tab, other_servers := Servers}) -> ets_delete_keys(Tab, Keys), %% Insert to other nodes and block till written - Monitors = delete_from_remote_nodes(Servers, Keys, FromPid), + Monitors = replicate(Servers, remote_delete, Keys, FromPid), {reply, {ok, Monitors}, State}. -delete_from_remote_nodes([{RemotePid, ProxyPid} | Servers], Keys, FromPid) -> +replicate([{RemotePid, ProxyPid} | Servers], Cmd, Payload, FromPid) -> Mon = erlang:monitor(process, ProxyPid), %% Reply would be routed directly to FromPid - Msg = {delete_from_remote_node, Mon, FromPid, Keys}, + Msg = {Cmd, Mon, FromPid, Payload}, send_to_remote(RemotePid, Msg), - [Mon | delete_from_remote_nodes(Servers, Keys, FromPid)]; -delete_from_remote_nodes([], _Keys, _FromPid) -> + [Mon | replicate(Servers, Cmd, Payload, FromPid)]; +replicate([], _Cmd, _Payload, _FromPid) -> []. apply_backlog([{Msg, From}|Backlog], State) -> From af27a2f3fdc9ef3afe04daf43c93e39bc0d1bd6c Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 4 Mar 2022 16:45:36 +0100 Subject: [PATCH 22/64] Reorder clauses in handle_call --- src/kiss.erl | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 2c6a3935..f4e8d999 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -107,20 +107,19 @@ init([Tab, Opts]) -> {ok, #{tab => Tab, other_servers => [], opts => Opts, backlog => [], paused => false, pause_monitor => undefined}}. - handle_call({insert, Rec}, From, State = #{paused := false}) -> handle_insert(Rec, From, State); handle_call({delete, Keys}, From, State = #{paused := false}) -> handle_delete(Keys, From, State); -handle_call(remote_dump, _From, State = #{tab := Tab}) -> - {reply, {ok, dump(Tab)}, State}; handle_call(other_servers, _From, State = #{other_servers := Servers}) -> {reply, Servers, State}; -handle_call(sync, _From, State) -> - handle_sync(State), +handle_call(sync, _From, State = #{other_servers := Servers}) -> + [ping(Pid) || Pid <- servers_to_pids(Servers)], {reply, ok, State}; handle_call(ping, _From, State) -> {reply, ping, State}; +handle_call(remote_dump, _From, State = #{tab := Tab}) -> + {reply, {ok, dump(Tab)}, State}; handle_call({send_dump_to_remote_node, NewPids, Dump}, _From, State) -> handle_send_dump_to_remote_node(NewPids, Dump, State); handle_call(pause, _From = {FromPid, _}, State) -> @@ -259,7 +258,3 @@ handle_unpause(State = #{backlog := Backlog, pause_monitor := Mon}) -> erlang:demonitor(Mon, [flush]), State2 = State#{pause => false, backlog := [], pause_monitor => undefined}, {reply, ok, apply_backlog(lists:reverse(Backlog), State2)}. - -handle_sync(#{other_servers := Servers}) -> - [ping(Pid) || Pid <- servers_to_pids(Servers)], - ok. From 8615e892e8a13f8ee7a76a581be657799f0d0fe0 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 4 Mar 2022 16:48:14 +0100 Subject: [PATCH 23/64] Fix stop output in kiss_proxy --- src/kiss_proxy.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kiss_proxy.erl b/src/kiss_proxy.erl index aacca0af..dcf24331 100644 --- a/src/kiss_proxy.erl +++ b/src/kiss_proxy.erl @@ -24,10 +24,10 @@ handle_cast(_Msg, State) -> handle_info({'DOWN', MonRef, process, Pid, _Reason}, State = #{mon := MonRef}) -> ?LOG_ERROR(#{what => node_down, remote_pid => Pid, node => node(Pid)}), - {stop, State}; + {stop, normal, State}; handle_info({'DOWN', MonRef, process, Pid, _Reason}, State = #{pmon := MonRef}) -> ?LOG_ERROR(#{what => parent_process_down, parent_pid => Pid}), - {stop, State}. + {stop, normal, State}. terminate(_Reason, _State) -> ok. From bf1c1e0b4dbdd064b5f7cb2763bd3ac13c05b504 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 4 Mar 2022 17:00:06 +0100 Subject: [PATCH 24/64] Reorder functions for better readability --- src/kiss.erl | 82 +++++++++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 36 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index f4e8d999..0b5987e8 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -17,12 +17,13 @@ -export([dump/1, remote_dump/1, send_dump_to_remote_node/3]). -export([other_nodes/1, other_pids/1]). -export([pause/1, unpause/1, sync/1]). - -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -include_lib("kernel/include/logger.hrl"). +%% API functions + %% Table and server has the same name %% Opts: %% - handle_down = fun(#{remote_pid => Pid, table => Tab}) @@ -40,8 +41,8 @@ stop(Tab) -> dump(Tab) -> ets:tab2list(Tab). -remote_dump(RemotePid) -> - short_call(RemotePid, remote_dump). +remote_dump(Pid) -> + short_call(Pid, remote_dump). send_dump_to_remote_node(RemotePid, NewPids, OurDump) -> Msg = {send_dump_to_remote_node, NewPids, OurDump}, @@ -64,16 +65,6 @@ delete_many(Server, Keys) -> {ok, Monitors} = gen_server:call(Server, {delete, Keys}), wait_for_updated(Monitors). -wait_for_updated([Mon | Monitors]) -> - receive - {updated, Mon} -> - wait_for_updated(Monitors); - {'DOWN', Mon, process, _Pid, _Reason} -> - wait_for_updated(Monitors) - end; -wait_for_updated([]) -> - ok. - other_servers(Server) -> gen_server:call(Server, other_servers). @@ -95,11 +86,7 @@ sync(RemotePid) -> ping(RemotePid) -> short_call(RemotePid, ping). -short_call(RemotePid, Msg) -> - F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, - Info = #{task => Msg, - remote_pid => RemotePid, remote_node => node(RemotePid)}, - kiss_long:run_safely(Info, F). +%% gen_server callbacks init([Tab, Opts]) -> ets:new(Tab, [ordered_set, named_table, @@ -150,13 +137,17 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -handle_send_dump_to_remote_node(NewPids, Dump, State = #{tab := Tab, other_servers := Servers}) -> +%% Internal logic + +handle_send_dump_to_remote_node(NewPids, Dump, + State = #{tab := Tab, other_servers := Servers}) -> ets:insert(Tab, Dump), Servers2 = add_servers(NewPids, Servers), {reply, ok, State#{other_servers => Servers2}}. handle_down(Mon, PausedByPid, State = #{pause_monitor := Mon}) -> - ?LOG_ERROR(#{what => pause_owner_crashed, state => State, paused_by_pid => PausedByPid}), + ?LOG_ERROR(#{what => pause_owner_crashed, + state => State, paused_by_pid => PausedByPid}), handle_unpause(State); handle_down(_Mon, ProxyPid, State = #{other_servers := Servers}) -> case lists:keytake(ProxyPid, 2, Servers) of @@ -166,7 +157,8 @@ handle_down(_Mon, ProxyPid, State = #{other_servers := Servers}) -> {noreply, State#{other_servers => Servers2}}; false -> %% This should not happen - ?LOG_ERROR(#{what => handle_down_failed, proxy_pid => ProxyPid, state => State}), + ?LOG_ERROR(#{what => handle_down_failed, + proxy_pid => ProxyPid, state => State}), {noreply, State} end. @@ -203,18 +195,6 @@ servers_to_pids(Servers) -> has_remote_pid(RemotePid, Servers) -> lists:keymember(RemotePid, 1, Servers). -%% Cleanup -call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> - case Opts of - #{handle_down := F} -> - FF = fun() -> F(#{remote_pid => RemotePid, table => Tab}) end, - Info = #{task => call_user_handle_down, table => Tab, - remote_pid => RemotePid, remote_node => node(RemotePid)}, - kiss_long:run_safely(Info, FF); - _ -> - ok - end. - reply_updated(Pid, Mon) -> %% We really don't wanna block this process erlang:send(Pid, {updated, Mon}, [noconnect, nosuspend]). @@ -222,13 +202,15 @@ reply_updated(Pid, Mon) -> send_to_remote(RemotePid, Msg) -> erlang:send(RemotePid, Msg, [noconnect, nosuspend]). -handle_insert(Rec, _From = {FromPid, _}, State = #{tab := Tab, other_servers := Servers}) -> +handle_insert(Rec, _From = {FromPid, _}, + State = #{tab := Tab, other_servers := Servers}) -> ets:insert(Tab, Rec), %% Insert to other nodes and block till written Monitors = replicate(Servers, remote_insert, Rec, FromPid), {reply, {ok, Monitors}, State}. -handle_delete(Keys, _From = {FromPid, _}, State = #{tab := Tab, other_servers := Servers}) -> +handle_delete(Keys, _From = {FromPid, _}, + State = #{tab := Tab, other_servers := Servers}) -> ets_delete_keys(Tab, Keys), %% Insert to other nodes and block till written Monitors = replicate(Servers, remote_delete, Keys, FromPid), @@ -243,13 +225,29 @@ replicate([{RemotePid, ProxyPid} | Servers], Cmd, Payload, FromPid) -> replicate([], _Cmd, _Payload, _FromPid) -> []. -apply_backlog([{Msg, From}|Backlog], State) -> +wait_for_updated([Mon | Monitors]) -> + receive + {updated, Mon} -> + wait_for_updated(Monitors); + {'DOWN', Mon, process, _Pid, _Reason} -> + wait_for_updated(Monitors) + end; +wait_for_updated([]) -> + ok. + +apply_backlog([{Msg, From} | Backlog], State) -> {reply, Reply, State2} = handle_call(Msg, From, State), gen_server:reply(From, Reply), apply_backlog(Backlog, State2); apply_backlog([], State) -> State. +short_call(RemotePid, Msg) -> + F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, + Info = #{task => Msg, + remote_pid => RemotePid, remote_node => node(RemotePid)}, + kiss_long:run_safely(Info, F). + %% Theoretically we can support mupltiple pauses (but no need for now because %% we pause in the global locked function) handle_unpause(State = #{pause := false}) -> @@ -258,3 +256,15 @@ handle_unpause(State = #{backlog := Backlog, pause_monitor := Mon}) -> erlang:demonitor(Mon, [flush]), State2 = State#{pause => false, backlog := [], pause_monitor => undefined}, {reply, ok, apply_backlog(lists:reverse(Backlog), State2)}. + +%% Cleanup +call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> + case Opts of + #{handle_down := F} -> + FF = fun() -> F(#{remote_pid => RemotePid, table => Tab}) end, + Info = #{task => call_user_handle_down, table => Tab, + remote_pid => RemotePid, remote_node => node(RemotePid)}, + kiss_long:run_safely(Info, FF); + _ -> + ok + end. From f90ab5336ff2161048c864b73a5204a06c65a3f4 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Sat, 5 Mar 2022 22:16:44 +0100 Subject: [PATCH 25/64] Add events_are_applied_in_the_correct_order_after_unpause test case Add request versions of insert/delete functions --- src/kiss.erl | 27 +++++++++++++++++++++++---- test/kiss_SUITE.erl | 21 ++++++++++++++++++++- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 0b5987e8..b6d94027 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -19,6 +19,7 @@ -export([pause/1, unpause/1, sync/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-export([insert_request/2, delete_request/2, delete_many_request/2, wait_response/2]). -include_lib("kernel/include/logger.hrl"). @@ -65,6 +66,23 @@ delete_many(Server, Keys) -> {ok, Monitors} = gen_server:call(Server, {delete, Keys}), wait_for_updated(Monitors). +insert_request(Server, Rec) -> + gen_server:send_request(Server, {insert, Rec}). + +delete_request(Tab, Key) -> + delete_many_request(Tab, [Key]). + +delete_many_request(Server, Keys) -> + gen_server:send_request(Server, {delete, Keys}). + +wait_response(RequestId, Timeout) -> + case gen_server:wait_response(RequestId, Timeout) of + {reply, {ok, Monitors}} -> + wait_for_updated(Monitors); + Other -> + Other + end. + other_servers(Server) -> gen_server:call(Server, other_servers). @@ -111,7 +129,7 @@ handle_call({send_dump_to_remote_node, NewPids, Dump}, _From, State) -> handle_send_dump_to_remote_node(NewPids, Dump, State); handle_call(pause, _From = {FromPid, _}, State) -> Mon = erlang:monitor(process, FromPid), - {reply, ok, State#{pause => true, pause_monitor => Mon}}; + {reply, ok, State#{paused => true, pause_monitor => Mon}}; handle_call(unpause, _From, State) -> handle_unpause(State); handle_call(Msg, From, State = #{paused := true, backlog := Backlog}) -> @@ -148,7 +166,8 @@ handle_send_dump_to_remote_node(NewPids, Dump, handle_down(Mon, PausedByPid, State = #{pause_monitor := Mon}) -> ?LOG_ERROR(#{what => pause_owner_crashed, state => State, paused_by_pid => PausedByPid}), - handle_unpause(State); + {reply, ok, State2} = handle_unpause(State), + {noreply, State2}; handle_down(_Mon, ProxyPid, State = #{other_servers := Servers}) -> case lists:keytake(ProxyPid, 2, Servers) of {value, {RemotePid, _}, Servers2} -> @@ -250,11 +269,11 @@ short_call(RemotePid, Msg) -> %% Theoretically we can support mupltiple pauses (but no need for now because %% we pause in the global locked function) -handle_unpause(State = #{pause := false}) -> +handle_unpause(State = #{paused := false}) -> {reply, {error, already_unpaused}, State}; handle_unpause(State = #{backlog := Backlog, pause_monitor := Mon}) -> erlang:demonitor(Mon, [flush]), - State2 = State#{pause => false, backlog := [], pause_monitor => undefined}, + State2 = State#{paused => false, backlog := [], pause_monitor => undefined}, {reply, ok, apply_backlog(lists:reverse(Backlog), State2)}. %% Cleanup diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index 5702f23f..97eb5325 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -5,7 +5,8 @@ all() -> [test_multinode, node_list_is_correct, test_multinode_auto_discovery, test_locally, - handle_down_is_called]. + handle_down_is_called, + events_are_applied_in_the_correct_order_after_unpause]. init_per_suite(Config) -> Node2 = start_node(ct2), @@ -121,6 +122,24 @@ handle_down_is_called(_Config) -> after 5000 -> ct:fail(timeout) end. +events_are_applied_in_the_correct_order_after_unpause(_Config) -> + T = t4, + {ok, Pid} = kiss:start(T, #{}), + ok = kiss:pause(Pid), + R1 = kiss:insert_request(T, {1}), + R2 = kiss:delete_request(T, 1), + kiss:delete_request(T, 2), + kiss:insert_request(T, {2}), + kiss:insert_request(T, {3}), + kiss:insert_request(T, {4}), + kiss:insert_request(T, {5}), + R3 = kiss:insert_request(T, [{6}, {7}]), + R4 = kiss:delete_many_request(T, [5, 4]), + [] = lists:sort(kiss:dump(T)), + ok = kiss:unpause(Pid), + [ok = kiss:wait_response(R, 5000) || R <- [R1, R2, R3, R4]], + [{2}, {3}, {6}, {7}] = lists:sort(kiss:dump(T)). + start(Node, Tab) -> rpc(Node, kiss, start, [Tab, #{}]). From ae398739cc30dd645458893eda78ac42e910f186 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Sun, 6 Mar 2022 20:16:52 +0100 Subject: [PATCH 26/64] Use monitor table --- src/kiss.erl | 111 +++++++++++++++++++++++++++----------------- src/kiss_proxy.erl | 37 --------------- test/kiss_SUITE.erl | 12 ++++- 3 files changed, 79 insertions(+), 81 deletions(-) delete mode 100644 src/kiss_proxy.erl diff --git a/src/kiss.erl b/src/kiss.erl index b6d94027..2a5fb673 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -2,7 +2,6 @@ %% One file, everything is simple, but we don't silently hide race conditions %% No transactions %% We don't use rpc module, because it is one gen_server -%% We monitor a proxy module (so, no remote monitors on each insert) %% If we write in format {Key, WriterName}, we should resolve conflicts automatically. @@ -63,8 +62,8 @@ delete(Tab, Key) -> %% A separate function for multidelete (because key COULD be a list, so no confusion) delete_many(Server, Keys) -> - {ok, Monitors} = gen_server:call(Server, {delete, Keys}), - wait_for_updated(Monitors). + {ok, WaitInfo} = gen_server:call(Server, {delete, Keys}), + wait_for_updated(WaitInfo). insert_request(Server, Rec) -> gen_server:send_request(Server, {insert, Rec}). @@ -77,8 +76,8 @@ delete_many_request(Server, Keys) -> wait_response(RequestId, Timeout) -> case gen_server:wait_response(RequestId, Timeout) of - {reply, {ok, Monitors}} -> - wait_for_updated(Monitors); + {reply, {ok, WaitInfo}} -> + wait_for_updated(WaitInfo); Other -> Other end. @@ -107,9 +106,11 @@ ping(RemotePid) -> %% gen_server callbacks init([Tab, Opts]) -> - ets:new(Tab, [ordered_set, named_table, - public, {read_concurrency, true}]), - {ok, #{tab => Tab, other_servers => [], opts => Opts, backlog => [], + MonTabName = list_to_atom(atom_to_list(Tab) ++ "_mon"), + ets:new(Tab, [ordered_set, named_table, public]), + MonTab = ets:new(MonTabName, [public]), + {ok, #{tab => Tab, mon_tab => MonTab, + other_servers => [], opts => Opts, backlog => [], paused => false, pause_monitor => undefined}}. handle_call({insert, Rec}, From, State = #{paused := false}) -> @@ -168,35 +169,45 @@ handle_down(Mon, PausedByPid, State = #{pause_monitor := Mon}) -> state => State, paused_by_pid => PausedByPid}), {reply, ok, State2} = handle_unpause(State), {noreply, State2}; -handle_down(_Mon, ProxyPid, State = #{other_servers := Servers}) -> - case lists:keytake(ProxyPid, 2, Servers) of - {value, {RemotePid, _}, Servers2} -> +handle_down(Mon, RemotePid, State = #{other_servers := Servers, mon_tab := MonTab}) -> + case lists:member(RemotePid, Servers) of + true -> + Servers2 = lists:delete(RemotePid, Servers), + notify_remote_down(RemotePid, MonTab), %% Down from a proxy call_user_handle_down(RemotePid, State), {noreply, State#{other_servers => Servers2}}; false -> - %% This should not happen - ?LOG_ERROR(#{what => handle_down_failed, - proxy_pid => ProxyPid, state => State}), + %% Caller process is DOWN + ets:delete(MonTab, Mon), {noreply, State} end. +notify_remote_down(RemotePid, MonTab) -> + List = ets:tab2list(MonTab), + notify_remote_down_loop(RemotePid, List). + +notify_remote_down_loop(RemotePid, [{Mon, Pid} | List]) -> + Pid ! {'DOWN', Mon, process, RemotePid, notify_remote_down}, + notify_remote_down_loop(RemotePid, List); +notify_remote_down_loop(_RemotePid, []) -> + ok. + add_servers(Pids, Servers) -> - lists:sort(start_proxies_for(Pids, Servers) ++ Servers). + lists:sort(add_servers2(Pids, Servers) ++ Servers). -start_proxies_for([RemotePid | OtherPids], Servers) +add_servers2([RemotePid | OtherPids], Servers) when is_pid(RemotePid), RemotePid =/= self() -> case has_remote_pid(RemotePid, Servers) of false -> - {ok, ProxyPid} = kiss_proxy:start(RemotePid), - erlang:monitor(process, ProxyPid), - [{RemotePid, ProxyPid} | start_proxies_for(OtherPids, Servers)]; + erlang:monitor(process, RemotePid), + [RemotePid | add_servers2(OtherPids, Servers)]; true -> ?LOG_INFO(#{what => already_added, remote_pid => RemotePid, remote_node => node(RemotePid)}), - start_proxies_for(OtherPids, Servers) + add_servers2(OtherPids, Servers) end; -start_proxies_for([], _Servers) -> +add_servers2([], _Servers) -> []. pids_to_nodes(Pids) -> @@ -209,50 +220,64 @@ ets_delete_keys(_Tab, []) -> ok. servers_to_pids(Servers) -> - [Pid || {Pid, _} <- Servers]. + [Pid || Pid <- Servers]. has_remote_pid(RemotePid, Servers) -> - lists:keymember(RemotePid, 1, Servers). + lists:member(RemotePid, Servers). reply_updated(Pid, Mon) -> %% We really don't wanna block this process - erlang:send(Pid, {updated, Mon}, [noconnect, nosuspend]). + erlang:send(Pid, {updated, Mon, self()}, [noconnect, nosuspend]). send_to_remote(RemotePid, Msg) -> erlang:send(RemotePid, Msg, [noconnect, nosuspend]). handle_insert(Rec, _From = {FromPid, _}, - State = #{tab := Tab, other_servers := Servers}) -> + State = #{tab := Tab, mon_tab := MonTab, other_servers := Servers}) -> ets:insert(Tab, Rec), %% Insert to other nodes and block till written - Monitors = replicate(Servers, remote_insert, Rec, FromPid), - {reply, {ok, Monitors}, State}. + WaitInfo = replicate(Servers, remote_insert, Rec, FromPid, MonTab), + {reply, {ok, WaitInfo}, State}. handle_delete(Keys, _From = {FromPid, _}, - State = #{tab := Tab, other_servers := Servers}) -> + State = #{tab := Tab, mon_tab := MonTab, other_servers := Servers}) -> ets_delete_keys(Tab, Keys), %% Insert to other nodes and block till written - Monitors = replicate(Servers, remote_delete, Keys, FromPid), - {reply, {ok, Monitors}, State}. + WaitInfo = replicate(Servers, remote_delete, Keys, FromPid, MonTab), + {reply, {ok, WaitInfo}, State}. + +replicate(Servers, Cmd, Payload, FromPid, MonTab) -> + Mon = erlang:monitor(process, FromPid), + ets:insert(MonTab, {Mon, FromPid}), + replicate2(Mon, Servers, Cmd, Payload, FromPid), + {Mon, Servers, MonTab}. -replicate([{RemotePid, ProxyPid} | Servers], Cmd, Payload, FromPid) -> - Mon = erlang:monitor(process, ProxyPid), +replicate2(Mon, [RemotePid | Servers], Cmd, Payload, FromPid) -> %% Reply would be routed directly to FromPid Msg = {Cmd, Mon, FromPid, Payload}, send_to_remote(RemotePid, Msg), - [Mon | replicate(Servers, Cmd, Payload, FromPid)]; -replicate([], _Cmd, _Payload, _FromPid) -> - []. + replicate2(Mon, Servers, Cmd, Payload, FromPid); +replicate2(_Mon, [], _Cmd, _Payload, _FromPid) -> + ok. + +wait_for_updated({Mon, Servers, MonTab}) -> + try + wait_for_updated2(Mon, Servers) + after + ets:delete(MonTab, Mon) + end. -wait_for_updated([Mon | Monitors]) -> +wait_for_updated2(_Mon, []) -> + ok; +wait_for_updated2(Mon, Servers) -> receive - {updated, Mon} -> - wait_for_updated(Monitors); - {'DOWN', Mon, process, _Pid, _Reason} -> - wait_for_updated(Monitors) - end; -wait_for_updated([]) -> - ok. + {updated, Mon, Pid} -> + Servers2 = lists:delete(Pid, Servers), + wait_for_updated2(Mon, Servers2); + {'DOWN', Mon, process, Pid, _Reason} -> + Servers2 = lists:delete(Pid, Servers), + wait_for_updated2(Mon, Servers2) + end. apply_backlog([{Msg, From} | Backlog], State) -> {reply, Reply, State2} = handle_call(Msg, From, State), diff --git a/src/kiss_proxy.erl b/src/kiss_proxy.erl deleted file mode 100644 index dcf24331..00000000 --- a/src/kiss_proxy.erl +++ /dev/null @@ -1,37 +0,0 @@ -%% We monitor this process instead of a remote process --module(kiss_proxy). --behaviour(gen_server). - --export([start/1]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --include_lib("kernel/include/logger.hrl"). - -start(RemotePid) -> - gen_server:start(?MODULE, [RemotePid, self()], []). - -init([RemotePid, ParentPid]) -> - MonRef = erlang:monitor(process, RemotePid), - MonRef2 = erlang:monitor(process, ParentPid), - {ok, #{mon => MonRef, pmon => MonRef2, remote_pid => RemotePid}}. - -handle_call(_Reply, _From, State) -> - {reply, ok, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info({'DOWN', MonRef, process, Pid, _Reason}, State = #{mon := MonRef}) -> - ?LOG_ERROR(#{what => node_down, remote_pid => Pid, node => node(Pid)}), - {stop, normal, State}; -handle_info({'DOWN', MonRef, process, Pid, _Reason}, State = #{pmon := MonRef}) -> - ?LOG_ERROR(#{what => parent_process_down, parent_pid => Pid}), - {stop, normal, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index 97eb5325..dfdd8282 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -6,7 +6,8 @@ all() -> [test_multinode, node_list_is_correct, test_multinode_auto_discovery, test_locally, handle_down_is_called, - events_are_applied_in_the_correct_order_after_unpause]. + events_are_applied_in_the_correct_order_after_unpause, + write_returns_if_remote_server_crashes]. init_per_suite(Config) -> Node2 = start_node(ct2), @@ -140,6 +141,15 @@ events_are_applied_in_the_correct_order_after_unpause(_Config) -> [ok = kiss:wait_response(R, 5000) || R <- [R1, R2, R3, R4]], [{2}, {3}, {6}, {7}] = lists:sort(kiss:dump(T)). +write_returns_if_remote_server_crashes(_Config) -> + {ok, Pid1} = kiss:start(c1, #{}), + {ok, Pid2} = kiss:start(c2, #{}), + ok = kiss_join:join(lock1, #{table => [c1, c2]}, Pid1, Pid2), + sys:suspend(Pid2), + R = kiss:insert_request(c1, {1}), + exit(Pid2, oops), + ok = kiss:wait_response(R, 5000). + start(Node, Tab) -> rpc(Node, kiss, start, [Tab, #{}]). From 4d3116ca8d222297b515e22fa8bc51c8744407ce Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Sun, 6 Mar 2022 20:34:24 +0100 Subject: [PATCH 27/64] Add MonTab cleaner --- src/kiss.erl | 19 ++++++------ src/kiss_mon_cleaner.erl | 62 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 9 deletions(-) create mode 100644 src/kiss_mon_cleaner.erl diff --git a/src/kiss.erl b/src/kiss.erl index 2a5fb673..5c550153 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -109,6 +109,7 @@ init([Tab, Opts]) -> MonTabName = list_to_atom(atom_to_list(Tab) ++ "_mon"), ets:new(Tab, [ordered_set, named_table, public]), MonTab = ets:new(MonTabName, [public]), + kiss_mon_cleaner:start_link(MonTabName, MonTab), {ok, #{tab => Tab, mon_tab => MonTab, other_servers => [], opts => Opts, backlog => [], paused => false, pause_monitor => undefined}}. @@ -169,7 +170,7 @@ handle_down(Mon, PausedByPid, State = #{pause_monitor := Mon}) -> state => State, paused_by_pid => PausedByPid}), {reply, ok, State2} = handle_unpause(State), {noreply, State2}; -handle_down(Mon, RemotePid, State = #{other_servers := Servers, mon_tab := MonTab}) -> +handle_down(_Mon, RemotePid, State = #{other_servers := Servers, mon_tab := MonTab}) -> case lists:member(RemotePid, Servers) of true -> Servers2 = lists:delete(RemotePid, Servers), @@ -178,8 +179,9 @@ handle_down(Mon, RemotePid, State = #{other_servers := Servers, mon_tab := MonTa call_user_handle_down(RemotePid, State), {noreply, State#{other_servers => Servers2}}; false -> - %% Caller process is DOWN - ets:delete(MonTab, Mon), + %% This should not happen + ?LOG_ERROR(#{what => handle_down_failed, + remote_pid => RemotePid, state => State}), {noreply, State} end. @@ -232,22 +234,21 @@ reply_updated(Pid, Mon) -> send_to_remote(RemotePid, Msg) -> erlang:send(RemotePid, Msg, [noconnect, nosuspend]). -handle_insert(Rec, _From = {FromPid, _}, +handle_insert(Rec, _From = {FromPid, Mon}, State = #{tab := Tab, mon_tab := MonTab, other_servers := Servers}) -> ets:insert(Tab, Rec), %% Insert to other nodes and block till written - WaitInfo = replicate(Servers, remote_insert, Rec, FromPid, MonTab), + WaitInfo = replicate(Mon, Servers, remote_insert, Rec, FromPid, MonTab), {reply, {ok, WaitInfo}, State}. -handle_delete(Keys, _From = {FromPid, _}, +handle_delete(Keys, _From = {FromPid, Mon}, State = #{tab := Tab, mon_tab := MonTab, other_servers := Servers}) -> ets_delete_keys(Tab, Keys), %% Insert to other nodes and block till written - WaitInfo = replicate(Servers, remote_delete, Keys, FromPid, MonTab), + WaitInfo = replicate(Mon, Servers, remote_delete, Keys, FromPid, MonTab), {reply, {ok, WaitInfo}, State}. -replicate(Servers, Cmd, Payload, FromPid, MonTab) -> - Mon = erlang:monitor(process, FromPid), +replicate(Mon, Servers, Cmd, Payload, FromPid, MonTab) -> ets:insert(MonTab, {Mon, FromPid}), replicate2(Mon, Servers, Cmd, Payload, FromPid), {Mon, Servers, MonTab}. diff --git a/src/kiss_mon_cleaner.erl b/src/kiss_mon_cleaner.erl new file mode 100644 index 00000000..71e92667 --- /dev/null +++ b/src/kiss_mon_cleaner.erl @@ -0,0 +1,62 @@ +%% Monitor Table contains processes, that are waiting for writes to finish. +%% It is usually cleaned automatically. +%% Unless the caller process crashes. +%% This server removes such entries from the MonTab. +%% We don't expect the MonTab to be extremely big, so this check should be quick. +-module(kiss_mon_cleaner). +-behaviour(gen_server). + +-export([start_link/2]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-include_lib("kernel/include/logger.hrl"). + +start_link(Name, MonTab) -> + gen_server:start_link({local, Name}, ?MODULE, [MonTab], []). + +init([MonTab]) -> + State = #{mon_tab => MonTab, interval => 30000}, + schedule_check(State), + {ok, State}. + +handle_call(_Reply, _From, State) -> + {reply, ok, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(check, State) -> + handle_check(State), + {stop, normal, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +schedule_check(State = #{interval := Interval}) -> + cancel_old_timer(State), + TimerRef = erlang:send_after(Interval, self(), check), + State#{timer_ref => TimerRef}. + +cancel_old_timer(#{timer_ref := OldRef}) -> + erlang:cancel_timer(OldRef); +cancel_old_timer(_State) -> + ok. + +handle_check(State = #{mon_tab := MonTab}) -> + check_loop(ets:tab2list(MonTab), MonTab), + schedule_check(State). + +check_loop([{Mon, Pid} | List], MonTab) -> + case is_process_alive(Pid) of + true -> + ets:delete(MonTab, Mon); + false -> + ok + end, + check_loop(List, MonTab); +check_loop([], _MonTab) -> + ok. From e518ae87c6699c2a052e6804c7e01bc36fef4cc3 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Sun, 6 Mar 2022 20:39:42 +0100 Subject: [PATCH 28/64] Send remote_down from the local server --- src/kiss.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 5c550153..5d54a69e 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -190,7 +190,7 @@ notify_remote_down(RemotePid, MonTab) -> notify_remote_down_loop(RemotePid, List). notify_remote_down_loop(RemotePid, [{Mon, Pid} | List]) -> - Pid ! {'DOWN', Mon, process, RemotePid, notify_remote_down}, + Pid ! {remote_down, Mon, RemotePid}, notify_remote_down_loop(RemotePid, List); notify_remote_down_loop(_RemotePid, []) -> ok. @@ -275,7 +275,7 @@ wait_for_updated2(Mon, Servers) -> {updated, Mon, Pid} -> Servers2 = lists:delete(Pid, Servers), wait_for_updated2(Mon, Servers2); - {'DOWN', Mon, process, Pid, _Reason} -> + {remote_down, Mon, Pid} -> Servers2 = lists:delete(Pid, Servers), wait_for_updated2(Mon, Servers2) end. From bdcdc0f02566272e9909f14bc5cd1c98363c4074 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Sun, 6 Mar 2022 20:50:31 +0100 Subject: [PATCH 29/64] Add description into the file header --- src/kiss.erl | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 5d54a69e..8a6fa06b 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -1,14 +1,19 @@ -%% Very simple multinode ETS writer -%% One file, everything is simple, but we don't silently hide race conditions -%% No transactions -%% We don't use rpc module, because it is one gen_server - - -%% If we write in format {Key, WriterName}, we should resolve conflicts automatically. -%% +%% Very simple multinode ETS writer. +%% One file, everything is simple, but we don't silently hide race conditions. +%% No transactions support. +%% We don't use rpc module, because it is a single gen_server. +%% We use MonTab table instead of monitors to detect if one of remote servers +%% is down and would not send a replication result. %% While Tab is an atom, we can join tables with different atoms for the local testing. - -%% We don't use monitors to avoid round-trips (that's why we don't use calls neither) +%% We pause writes when a new node is joining (we resume them again). It is to +%% ensure that all writes would be bulk copied. +%% We support merging data on join by default. +%% We do not check if we override data during join So, it is up to the user +%% to ensure that merging would survive overrides. Two ways to do it: +%% - Write each key once and only once (basically add a reference into a key) +%% - Add writer pid() or writer node() as a key. And do a proper cleanups using handle_down. +%% (the data could still get overwritten though if a node joins back way too quick +%% and cleaning is done outside of handle_down) -module(kiss). -behaviour(gen_server). @@ -249,8 +254,8 @@ handle_delete(Keys, _From = {FromPid, Mon}, {reply, {ok, WaitInfo}, State}. replicate(Mon, Servers, Cmd, Payload, FromPid, MonTab) -> - ets:insert(MonTab, {Mon, FromPid}), replicate2(Mon, Servers, Cmd, Payload, FromPid), + ets:insert(MonTab, {Mon, FromPid}), {Mon, Servers, MonTab}. replicate2(Mon, [RemotePid | Servers], Cmd, Payload, FromPid) -> From 9bfdbfa0ea8a094b2dd241ec8fd72d53d9074d8b Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Sun, 6 Mar 2022 21:11:50 +0100 Subject: [PATCH 30/64] Simplify replicate2 --- src/kiss.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 8a6fa06b..84039db6 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -254,16 +254,16 @@ handle_delete(Keys, _From = {FromPid, Mon}, {reply, {ok, WaitInfo}, State}. replicate(Mon, Servers, Cmd, Payload, FromPid, MonTab) -> - replicate2(Mon, Servers, Cmd, Payload, FromPid), + %% Reply would be routed directly to FromPid + Msg = {Cmd, Mon, FromPid, Payload}, + replicate2(Servers, Msg), ets:insert(MonTab, {Mon, FromPid}), {Mon, Servers, MonTab}. -replicate2(Mon, [RemotePid | Servers], Cmd, Payload, FromPid) -> - %% Reply would be routed directly to FromPid - Msg = {Cmd, Mon, FromPid, Payload}, +replicate2([RemotePid | Servers], Msg) -> send_to_remote(RemotePid, Msg), - replicate2(Mon, Servers, Cmd, Payload, FromPid); -replicate2(_Mon, [], _Cmd, _Payload, _FromPid) -> + replicate2(Servers, Msg); +replicate2([], _Msg) -> ok. wait_for_updated({Mon, Servers, MonTab}) -> From 175b926dba5cccbd07e91eb96b5d6f8b51109888 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Sun, 6 Mar 2022 21:30:09 +0100 Subject: [PATCH 31/64] Add mon_cleaner_works testcase --- src/kiss.erl | 4 ++-- src/kiss_mon_cleaner.erl | 6 +++--- test/kiss_SUITE.erl | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 84039db6..693fc4c5 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -20,7 +20,7 @@ -export([start/2, stop/1, insert/2, delete/2, delete_many/2]). -export([dump/1, remote_dump/1, send_dump_to_remote_node/3]). -export([other_nodes/1, other_pids/1]). --export([pause/1, unpause/1, sync/1]). +-export([pause/1, unpause/1, sync/1, ping/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -export([insert_request/2, delete_request/2, delete_many_request/2, wait_response/2]). @@ -113,7 +113,7 @@ ping(RemotePid) -> init([Tab, Opts]) -> MonTabName = list_to_atom(atom_to_list(Tab) ++ "_mon"), ets:new(Tab, [ordered_set, named_table, public]), - MonTab = ets:new(MonTabName, [public]), + MonTab = ets:new(MonTabName, [public, named_table]), kiss_mon_cleaner:start_link(MonTabName, MonTab), {ok, #{tab => Tab, mon_tab => MonTab, other_servers => [], opts => Opts, backlog => [], diff --git a/src/kiss_mon_cleaner.erl b/src/kiss_mon_cleaner.erl index 71e92667..fb601979 100644 --- a/src/kiss_mon_cleaner.erl +++ b/src/kiss_mon_cleaner.erl @@ -28,7 +28,7 @@ handle_cast(_Msg, State) -> handle_info(check, State) -> handle_check(State), - {stop, normal, State}. + {noreply, State}. terminate(_Reason, _State) -> ok. @@ -52,9 +52,9 @@ handle_check(State = #{mon_tab := MonTab}) -> check_loop([{Mon, Pid} | List], MonTab) -> case is_process_alive(Pid) of - true -> - ets:delete(MonTab, Mon); false -> + ets:delete(MonTab, Mon); + true -> ok end, check_loop(List, MonTab); diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index dfdd8282..9f99614a 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -7,7 +7,8 @@ all() -> [test_multinode, node_list_is_correct, test_multinode_auto_discovery, test_locally, handle_down_is_called, events_are_applied_in_the_correct_order_after_unpause, - write_returns_if_remote_server_crashes]. + write_returns_if_remote_server_crashes, + mon_cleaner_works]. init_per_suite(Config) -> Node2 = start_node(ct2), @@ -150,6 +151,36 @@ write_returns_if_remote_server_crashes(_Config) -> exit(Pid2, oops), ok = kiss:wait_response(R, 5000). +mon_cleaner_works(_Config) -> + {ok, Pid1} = kiss:start(c3, #{}), + %% Suspend, so to avoid unexpected check + sys:suspend(c3_mon), + %% Two cases to check: an alive process and a dead process + R = kiss:insert_request(c3, {2}), + %% Ensure insert_request reaches the server + kiss:ping(Pid1), + %% There is one monitor + [_] = ets:tab2list(c3_mon), + {Pid, Mon} = spawn_monitor(fun() -> kiss:insert_request(c3, {1}) end), + receive + {'DOWN', Mon, process, Pid, _Reason} -> ok + after 5000 -> ct:fail(timeout) + end, + %% Ensure insert_request reaches the server + kiss:ping(Pid1), + %% There are two monitors + [_, _] = ets:tab2list(c3_mon), + %% Force check + sys:resume(c3_mon), + c3_mon ! check, + %% Ensure, that check is finished + sys:get_state(c3_mon), + %% A monitor for a dead process is removed + [_] = ets:tab2list(c3_mon), + %% The monitor is finally removed once wait_response returns + ok = kiss:wait_response(R, 5000), + [] = ets:tab2list(c3_mon). + start(Node, Tab) -> rpc(Node, kiss, start, [Tab, #{}]). From e3c3fb796cf7cb2c33276fd46a354e3fcbfc7810 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Sun, 6 Mar 2022 23:10:48 +0100 Subject: [PATCH 32/64] Add info function --- src/kiss.erl | 19 ++++++++++++++++--- src/kiss_discovery.erl | 11 ++++++++++- test/kiss_SUITE.erl | 2 ++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 693fc4c5..71698519 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -21,6 +21,7 @@ -export([dump/1, remote_dump/1, send_dump_to_remote_node/3]). -export([other_nodes/1, other_pids/1]). -export([pause/1, unpause/1, sync/1, ping/1]). +-export([info/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -export([insert_request/2, delete_request/2, delete_many_request/2, wait_response/2]). @@ -108,13 +109,16 @@ sync(RemotePid) -> ping(RemotePid) -> short_call(RemotePid, ping). +info(Server) -> + gen_server:call(Server, get_info). + %% gen_server callbacks init([Tab, Opts]) -> - MonTabName = list_to_atom(atom_to_list(Tab) ++ "_mon"), + MonTab = list_to_atom(atom_to_list(Tab) ++ "_mon"), ets:new(Tab, [ordered_set, named_table, public]), - MonTab = ets:new(MonTabName, [public, named_table]), - kiss_mon_cleaner:start_link(MonTabName, MonTab), + ets:new(MonTab, [public, named_table]), + kiss_mon_cleaner:start_link(MonTab, MonTab), {ok, #{tab => Tab, mon_tab => MonTab, other_servers => [], opts => Opts, backlog => [], paused => false, pause_monitor => undefined}}. @@ -139,6 +143,8 @@ handle_call(pause, _From = {FromPid, _}, State) -> {reply, ok, State#{paused => true, pause_monitor => Mon}}; handle_call(unpause, _From, State) -> handle_unpause(State); +handle_call(get_info, _From, State) -> + handle_get_info(State); handle_call(Msg, From, State = #{paused := true, backlog := Backlog}) -> {noreply, State#{backlog => [{Msg, From} | Backlog]}}. @@ -307,6 +313,13 @@ handle_unpause(State = #{backlog := Backlog, pause_monitor := Mon}) -> State2 = State#{paused => false, backlog := [], pause_monitor => undefined}, {reply, ok, apply_backlog(lists:reverse(Backlog), State2)}. +handle_get_info(State = #{tab := Tab, other_servers := Servers}) -> + Info = #{table => Tab, + nodes => lists:usort(pids_to_nodes([self() | Servers])), + size => ets:info(Tab, size), + memory => ets:info(Tab, memory)}, + {reply, Info, State}. + %% Cleanup call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> case Opts of diff --git a/src/kiss_discovery.erl b/src/kiss_discovery.erl index e0765bf4..a6de915f 100644 --- a/src/kiss_discovery.erl +++ b/src/kiss_discovery.erl @@ -2,7 +2,7 @@ -module(kiss_discovery). -behaviour(gen_server). --export([start/1, start_link/1, add_table/2]). +-export([start/1, start_link/1, add_table/2, info/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). @@ -33,6 +33,13 @@ start_common(F, Opts) -> add_table(Server, Table) -> gen_server:call(Server, {add_table, Table}). +get_tables(Server) -> + gen_server:call(Server, get_tables). + +info(Server) -> + {ok, Tables} = get_tables(Server), + [kiss:info(Tab) || Tab <- Tables]. + init(Opts) -> Mod = maps:get(backend_module, Opts, kiss_discovery_file), self() ! check, @@ -49,6 +56,8 @@ handle_call({add_table, Table}, _From, State = #{tables := Tables}) -> State2 = State#{tables => [Table | Tables]}, {reply, ok, handle_check(State2)} end; +handle_call(get_tables, _From, State = #{tables := Tables}) -> + {reply, {ok, Tables}, State}; handle_call(_Reply, _From, State) -> {reply, ok, State}. diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index 9f99614a..dd7c64ea 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -98,6 +98,8 @@ test_multinode_auto_discovery(Config) -> %% Waits for the first check ok = gen_server:call(Disco, ping), [Node2] = other_nodes(Node1, Tab), + [#{memory := _, nodes := [Node1, Node2], size := 0, table := tab2}] + = kiss_discovery:info(Disco), ok. test_locally(_Config) -> From 61b9d40327488c8e4ffc3f84c30d52fc5ae8e769 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 7 Mar 2022 09:56:44 +0100 Subject: [PATCH 33/64] Allow to pass atom to sync/1 --- src/kiss.erl | 6 ++++-- test/kiss_SUITE.erl | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 71698519..9cc1b3e5 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -298,11 +298,13 @@ apply_backlog([{Msg, From} | Backlog], State) -> apply_backlog([], State) -> State. -short_call(RemotePid, Msg) -> +short_call(RemotePid, Msg) when is_pid(RemotePid) -> F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, Info = #{task => Msg, remote_pid => RemotePid, remote_node => node(RemotePid)}, - kiss_long:run_safely(Info, F). + kiss_long:run_safely(Info, F); +short_call(Name, Msg) when is_atom(Name) -> + short_call(whereis(Name), Msg). %% Theoretically we can support mupltiple pauses (but no need for now because %% we pause in the global locked function) diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index dd7c64ea..1a9c3fe3 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -8,7 +8,7 @@ all() -> [test_multinode, node_list_is_correct, handle_down_is_called, events_are_applied_in_the_correct_order_after_unpause, write_returns_if_remote_server_crashes, - mon_cleaner_works]. + mon_cleaner_works, sync_using_name_works]. init_per_suite(Config) -> Node2 = start_node(ct2), @@ -183,6 +183,10 @@ mon_cleaner_works(_Config) -> ok = kiss:wait_response(R, 5000), [] = ets:tab2list(c3_mon). +sync_using_name_works(_Config) -> + {ok, _Pid1} = kiss:start(c4, #{}), + kiss:sync(c4). + start(Node, Tab) -> rpc(Node, kiss, start, [Tab, #{}]). From c4ce0ebb768562f5fb66b14ae582cd5c59aff581 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 7 Mar 2022 13:37:24 +0100 Subject: [PATCH 34/64] Log unexpected messages --- src/kiss.erl | 20 +++++++++++++++++--- src/kiss_discovery.erl | 16 ++++++++++------ src/kiss_mon_cleaner.erl | 11 ++++++++--- test/kiss_SUITE.erl | 2 +- 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 9cc1b3e5..20333da1 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -146,9 +146,16 @@ handle_call(unpause, _From, State) -> handle_call(get_info, _From, State) -> handle_get_info(State); handle_call(Msg, From, State = #{paused := true, backlog := Backlog}) -> - {noreply, State#{backlog => [{Msg, From} | Backlog]}}. + case should_backlogged(Msg) of + true -> + {noreply, State#{backlog => [{Msg, From} | Backlog]}}; + false -> + ?LOG_ERROR(#{what => unexpected_call, msg => Msg, from => From}), + {reply, {error, unexpected_call}, State} + end. -handle_cast(_Msg, State) -> +handle_cast(Msg, State) -> + ?LOG_ERROR(#{what => unexpected_cast, msg => Msg}), {noreply, State}. handle_info({remote_insert, Mon, Pid, Rec}, State = #{tab := Tab}) -> @@ -160,7 +167,10 @@ handle_info({remote_delete, Mon, Pid, Keys}, State = #{tab := Tab}) -> reply_updated(Pid, Mon), {noreply, State}; handle_info({'DOWN', Mon, process, Pid, _Reason}, State) -> - handle_down(Mon, Pid, State). + handle_down(Mon, Pid, State); +handle_info(Msg, State) -> + ?LOG_ERROR(#{what => unexpected_info, msg => Msg}), + {noreply, State}. terminate(_Reason, _State) -> ok. @@ -333,3 +343,7 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> _ -> ok end. + +should_backlogged({insert, _}) -> true; +should_backlogged({delete, _}) -> true; +should_backlogged(_) -> false. diff --git a/src/kiss_discovery.erl b/src/kiss_discovery.erl index a6de915f..edcba01f 100644 --- a/src/kiss_discovery.erl +++ b/src/kiss_discovery.erl @@ -58,14 +58,19 @@ handle_call({add_table, Table}, _From, State = #{tables := Tables}) -> end; handle_call(get_tables, _From, State = #{tables := Tables}) -> {reply, {ok, Tables}, State}; -handle_call(_Reply, _From, State) -> - {reply, ok, State}. +handle_call(Msg, From, State) -> + ?LOG_ERROR(#{what => unexpected_call, msg => Msg, from => From}), + {reply, {error, unexpected_call}, State}. -handle_cast(_Msg, State) -> +handle_cast(Msg, State) -> + ?LOG_ERROR(#{what => unexpected_cast, msg => Msg}), {noreply, State}. handle_info(check, State) -> - {noreply, handle_check(State)}. + {noreply, handle_check(State)}; +handle_info(Msg, State) -> + ?LOG_ERROR(#{what => unexpected_info, msg => Msg}), + {noreply, State}. terminate(_Reason, _State) -> ok. @@ -114,5 +119,4 @@ report_results(Results, _State = #{results := OldResults}) -> [report_result(Result) || Result <- Changed]. report_result(Map) -> - Text = [io_lib:format("~0p=~0p ", [K, V]) || {K, V} <- maps:to_list(Map)], - ?LOG_INFO(#{what => discovery_change, result => Text}). + ?LOG_INFO(Map). diff --git a/src/kiss_mon_cleaner.erl b/src/kiss_mon_cleaner.erl index fb601979..bb296394 100644 --- a/src/kiss_mon_cleaner.erl +++ b/src/kiss_mon_cleaner.erl @@ -20,14 +20,19 @@ init([MonTab]) -> schedule_check(State), {ok, State}. -handle_call(_Reply, _From, State) -> - {reply, ok, State}. +handle_call(Msg, From, State) -> + ?LOG_ERROR(#{what => unexpected_call, msg => Msg, from => From}), + {reply, {error, unexpected_call}, State}. -handle_cast(_Msg, State) -> +handle_cast(Msg, State) -> + ?LOG_ERROR(#{what => unexpected_cast, msg => Msg}), {noreply, State}. handle_info(check, State) -> handle_check(State), + {noreply, State}; +handle_info(Msg, State) -> + ?LOG_ERROR(#{what => unexpected_info, msg => Msg}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/test/kiss_SUITE.erl b/test/kiss_SUITE.erl index 1a9c3fe3..6d0935ea 100644 --- a/test/kiss_SUITE.erl +++ b/test/kiss_SUITE.erl @@ -96,7 +96,7 @@ test_multinode_auto_discovery(Config) -> ok = file:write_file(FileName, io_lib:format("~s~n~s~n", [Node1, Node2])), {ok, Disco} = kiss_discovery:start(#{tables => [Tab], disco_file => FileName}), %% Waits for the first check - ok = gen_server:call(Disco, ping), + sys:get_state(Disco), [Node2] = other_nodes(Node1, Tab), [#{memory := _, nodes := [Node1, Node2], size := 0, table := tab2}] = kiss_discovery:info(Disco), From 4125c47f111ffadbae47b25e4c1fcd8edc7718a6 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 9 Mar 2022 16:06:18 +0100 Subject: [PATCH 35/64] Add types --- src/kiss.erl | 72 +++++++++++++++++++++++++++++----------- src/kiss_discovery.erl | 27 +++++++++++---- src/kiss_mon_cleaner.erl | 26 ++++++++++----- 3 files changed, 91 insertions(+), 34 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index 20333da1..657a4355 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -28,6 +28,20 @@ -include_lib("kernel/include/logger.hrl"). +-type server() :: atom() | pid(). +-type request_id() :: term(). +-type backlog_msg() :: {insert, term()} | {delete, term()}. +-type from() :: {pid(), reference()}. +-type backlog_entry() :: {backlog_msg(), from()}. +-type state() :: #{ + tab := atom(), + mon_tab := atom(), + other_servers := [pid()], + opts := map(), + backlog := [backlog_entry()], + paused := false, + pause_monitor := undefined | reference()}. + %% API functions %% Table and server has the same name @@ -39,7 +53,7 @@ %% i.e. any functions that replicate changes are not allowed (i.e. insert/2, %% remove/2). start(Tab, Opts) when is_atom(Tab) -> - gen_server:start({local, Tab}, ?MODULE, [Tab, Opts], []). + gen_server:start({local, Tab}, ?MODULE, {Tab, Opts}, []). stop(Tab) -> gen_server:stop(Tab). @@ -47,8 +61,9 @@ stop(Tab) -> dump(Tab) -> ets:tab2list(Tab). -remote_dump(Pid) -> - short_call(Pid, remote_dump). +-spec remote_dump(server()) -> term(). +remote_dump(Server) -> + short_call(Server, remote_dump). send_dump_to_remote_node(RemotePid, NewPids, OurDump) -> Msg = {send_dump_to_remote_node, NewPids, OurDump}, @@ -59,27 +74,34 @@ send_dump_to_remote_node(RemotePid, NewPids, OurDump) -> %% Only the node that owns the data could update/remove the data. %% Ideally Key should contain inserter node info (for cleaning). +-spec insert(server(), tuple()) -> ok. insert(Server, Rec) -> {ok, Monitors} = gen_server:call(Server, {insert, Rec}), wait_for_updated(Monitors). -delete(Tab, Key) -> - delete_many(Tab, [Key]). +-spec delete(server(), term()) -> ok. +delete(Server, Key) -> + delete_many(Server, [Key]). %% A separate function for multidelete (because key COULD be a list, so no confusion) +-spec delete_many(server(), [term()]) -> ok. delete_many(Server, Keys) -> {ok, WaitInfo} = gen_server:call(Server, {delete, Keys}), wait_for_updated(WaitInfo). +-spec insert_request(server(), tuple()) -> request_id(). insert_request(Server, Rec) -> gen_server:send_request(Server, {insert, Rec}). +-spec delete_request(server(), term()) -> request_id(). delete_request(Tab, Key) -> delete_many_request(Tab, [Key]). +-spec delete_many_request(server(), term()) -> request_id(). delete_many_request(Server, Keys) -> gen_server:send_request(Server, {delete, Keys}). +-spec wait_response(request_id(), non_neg_integer() | timeout) -> term(). wait_response(RequestId, Timeout) -> case gen_server:wait_response(RequestId, Timeout) of {reply, {ok, WaitInfo}} -> @@ -94,27 +116,34 @@ other_servers(Server) -> other_nodes(Server) -> lists:usort(pids_to_nodes(other_pids(Server))). +-spec other_pids(server()) -> [pid()]. other_pids(Server) -> servers_to_pids(other_servers(Server)). -pause(RemotePid) -> - short_call(RemotePid, pause). +-spec pause(server()) -> term(). +pause(Server) -> + short_call(Server, pause). -unpause(RemotePid) -> - short_call(RemotePid, unpause). +-spec unpause(server()) -> term(). +unpause(Server) -> + short_call(Server, unpause). -sync(RemotePid) -> - short_call(RemotePid, sync). +-spec sync(server()) -> term(). +sync(Server) -> + short_call(Server, sync). -ping(RemotePid) -> - short_call(RemotePid, ping). +-spec ping(server()) -> term(). +ping(Server) -> + short_call(Server, ping). +-spec info(server()) -> term(). info(Server) -> gen_server:call(Server, get_info). %% gen_server callbacks -init([Tab, Opts]) -> +-spec init(term()) -> {ok, state()}. +init({Tab, Opts}) -> MonTab = list_to_atom(atom_to_list(Tab) ++ "_mon"), ets:new(Tab, [ordered_set, named_table, public]), ets:new(MonTab, [public, named_table]), @@ -123,6 +152,8 @@ init([Tab, Opts]) -> other_servers => [], opts => Opts, backlog => [], paused => false, pause_monitor => undefined}}. +-spec handle_call(term(), from(), state()) -> + {noreply, state()} | {reply, term(), state()}. handle_call({insert, Rec}, From, State = #{paused := false}) -> handle_insert(Rec, From, State); handle_call({delete, Keys}, From, State = #{paused := false}) -> @@ -140,7 +171,7 @@ handle_call({send_dump_to_remote_node, NewPids, Dump}, _From, State) -> handle_send_dump_to_remote_node(NewPids, Dump, State); handle_call(pause, _From = {FromPid, _}, State) -> Mon = erlang:monitor(process, FromPid), - {reply, ok, State#{paused => true, pause_monitor => Mon}}; + {reply, ok, State#{paused := true, pause_monitor := Mon}}; handle_call(unpause, _From, State) -> handle_unpause(State); handle_call(get_info, _From, State) -> @@ -148,16 +179,18 @@ handle_call(get_info, _From, State) -> handle_call(Msg, From, State = #{paused := true, backlog := Backlog}) -> case should_backlogged(Msg) of true -> - {noreply, State#{backlog => [{Msg, From} | Backlog]}}; + {noreply, State#{backlog := [{Msg, From} | Backlog]}}; false -> ?LOG_ERROR(#{what => unexpected_call, msg => Msg, from => From}), {reply, {error, unexpected_call}, State} end. +-spec handle_cast(term(), state()) -> {noreply, state()}. handle_cast(Msg, State) -> ?LOG_ERROR(#{what => unexpected_cast, msg => Msg}), {noreply, State}. +-spec handle_info(term(), state()) -> {noreply, state()}. handle_info({remote_insert, Mon, Pid, Rec}, State = #{tab := Tab}) -> ets:insert(Tab, Rec), reply_updated(Pid, Mon), @@ -184,7 +217,7 @@ handle_send_dump_to_remote_node(NewPids, Dump, State = #{tab := Tab, other_servers := Servers}) -> ets:insert(Tab, Dump), Servers2 = add_servers(NewPids, Servers), - {reply, ok, State#{other_servers => Servers2}}. + {reply, ok, State#{other_servers := Servers2}}. handle_down(Mon, PausedByPid, State = #{pause_monitor := Mon}) -> ?LOG_ERROR(#{what => pause_owner_crashed, @@ -198,7 +231,7 @@ handle_down(_Mon, RemotePid, State = #{other_servers := Servers, mon_tab := MonT notify_remote_down(RemotePid, MonTab), %% Down from a proxy call_user_handle_down(RemotePid, State), - {noreply, State#{other_servers => Servers2}}; + {noreply, State#{other_servers := Servers2}}; false -> %% This should not happen ?LOG_ERROR(#{what => handle_down_failed, @@ -308,6 +341,7 @@ apply_backlog([{Msg, From} | Backlog], State) -> apply_backlog([], State) -> State. +-spec short_call(server(), term()) -> term(). short_call(RemotePid, Msg) when is_pid(RemotePid) -> F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, Info = #{task => Msg, @@ -322,7 +356,7 @@ handle_unpause(State = #{paused := false}) -> {reply, {error, already_unpaused}, State}; handle_unpause(State = #{backlog := Backlog, pause_monitor := Mon}) -> erlang:demonitor(Mon, [flush]), - State2 = State#{paused => false, backlog := [], pause_monitor => undefined}, + State2 = State#{paused := false, backlog := [], pause_monitor := undefined}, {reply, ok, apply_backlog(lists:reverse(Backlog), State2)}. handle_get_info(State = #{tab := Tab, other_servers := Servers}) -> diff --git a/src/kiss_discovery.erl b/src/kiss_discovery.erl index edcba01f..069591b5 100644 --- a/src/kiss_discovery.erl +++ b/src/kiss_discovery.erl @@ -11,6 +11,15 @@ -type backend_state() :: term(). -type get_nodes_result() :: {ok, [node()]} | {error, term()}. +-type from() :: {pid(), reference()}. +-type state() :: #{ + results := [term()], + tables := [atom()], + backend_module := module(), + backend_state := state(), + timer_ref := reference() | undefined + }. + -callback init(map()) -> backend_state(). -callback get_nodes(backend_state()) -> {get_nodes_result(), backend_state()}. @@ -40,20 +49,23 @@ info(Server) -> {ok, Tables} = get_tables(Server), [kiss:info(Tab) || Tab <- Tables]. +-spec init(term()) -> {ok, state()}. init(Opts) -> Mod = maps:get(backend_module, Opts, kiss_discovery_file), self() ! check, Tables = maps:get(tables, Opts, []), BackendState = Mod:init(Opts), {ok, #{results => [], tables => Tables, - backend_module => Mod, backend_state => BackendState}}. + backend_module => Mod, backend_state => BackendState, + timer_ref => undefined}}. +-spec handle_call(term(), from(), state()) -> {reply, term(), state()}. handle_call({add_table, Table}, _From, State = #{tables := Tables}) -> case lists:member(Table, Tables) of true -> {reply, {error, already_added}, State}; false -> - State2 = State#{tables => [Table | Tables]}, + State2 = State#{tables := [Table | Tables]}, {reply, ok, handle_check(State2)} end; handle_call(get_tables, _From, State = #{tables := Tables}) -> @@ -62,10 +74,12 @@ handle_call(Msg, From, State) -> ?LOG_ERROR(#{what => unexpected_call, msg => Msg, from => From}), {reply, {error, unexpected_call}, State}. +-spec handle_cast(term(), state()) -> {noreply, state()}. handle_cast(Msg, State) -> ?LOG_ERROR(#{what => unexpected_cast, msg => Msg}), {noreply, State}. +-spec handle_info(term(), state()) -> {noreply, state()}. handle_info(check, State) -> {noreply, handle_check(State)}; handle_info(Msg, State) -> @@ -78,27 +92,28 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +-spec handle_check(state()) -> state(). handle_check(State = #{tables := []}) -> %% No tables to track, skip schedule_check(State); handle_check(State = #{backend_module := Mod, backend_state := BackendState}) -> {Res, BackendState2} = Mod:get_nodes(BackendState), State2 = handle_get_nodes_result(Res, State), - schedule_check(State2#{backend_state => BackendState2}). + schedule_check(State2#{backend_state := BackendState2}). handle_get_nodes_result({error, _Reason}, State) -> State; handle_get_nodes_result({ok, Nodes}, State = #{tables := Tables}) -> Results = [do_join(Tab, Node) || Tab <- Tables, Node <- Nodes, node() =/= Node], report_results(Results, State), - State#{results => Results}. + State#{results := Results}. schedule_check(State) -> cancel_old_timer(State), TimerRef = erlang:send_after(5000, self(), check), - State#{timer_ref => TimerRef}. + State#{timer_ref := TimerRef}. -cancel_old_timer(#{timer_ref := OldRef}) -> +cancel_old_timer(#{timer_ref := OldRef}) when is_reference(OldRef) -> erlang:cancel_timer(OldRef); cancel_old_timer(_State) -> ok. diff --git a/src/kiss_mon_cleaner.erl b/src/kiss_mon_cleaner.erl index bb296394..a79a250f 100644 --- a/src/kiss_mon_cleaner.erl +++ b/src/kiss_mon_cleaner.erl @@ -12,13 +12,20 @@ -include_lib("kernel/include/logger.hrl"). +-type timer_ref() :: reference() | undefined. +-type state() :: #{ + mon_tab := atom(), + interval := non_neg_integer(), + timer_ref := timer_ref() + }. + start_link(Name, MonTab) -> - gen_server:start_link({local, Name}, ?MODULE, [MonTab], []). + gen_server:start_link({local, Name}, ?MODULE, MonTab, []). -init([MonTab]) -> - State = #{mon_tab => MonTab, interval => 30000}, - schedule_check(State), - {ok, State}. +-spec init(atom()) -> {ok, state()}. +init(MonTab) -> + State = #{mon_tab => MonTab, interval => 30000, timer_ref => undefined}, + {ok, schedule_check(State)}. handle_call(Msg, From, State) -> ?LOG_ERROR(#{what => unexpected_call, msg => Msg, from => From}), @@ -29,8 +36,7 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info(check, State) -> - handle_check(State), - {noreply, State}; + {noreply, handle_check(State)}; handle_info(Msg, State) -> ?LOG_ERROR(#{what => unexpected_info, msg => Msg}), {noreply, State}. @@ -41,16 +47,18 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +-spec schedule_check(state()) -> state(). schedule_check(State = #{interval := Interval}) -> cancel_old_timer(State), TimerRef = erlang:send_after(Interval, self(), check), - State#{timer_ref => TimerRef}. + State#{timer_ref := TimerRef}. -cancel_old_timer(#{timer_ref := OldRef}) -> +cancel_old_timer(#{timer_ref := OldRef}) when is_reference(OldRef) -> erlang:cancel_timer(OldRef); cancel_old_timer(_State) -> ok. +-spec handle_check(state()) -> state(). handle_check(State = #{mon_tab := MonTab}) -> check_loop(ets:tab2list(MonTab), MonTab), schedule_check(State). From ae6cab5131a93e3654a63f60c9accfbe58fcd382 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 9 Mar 2022 19:41:14 +0100 Subject: [PATCH 36/64] Set message_queue_data=off_heap --- src/kiss.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/kiss.erl b/src/kiss.erl index 657a4355..f18e7c28 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -144,6 +144,7 @@ info(Server) -> -spec init(term()) -> {ok, state()}. init({Tab, Opts}) -> + process_flag(message_queue_data, off_heap), MonTab = list_to_atom(atom_to_list(Tab) ++ "_mon"), ets:new(Tab, [ordered_set, named_table, public]), ets:new(MonTab, [public, named_table]), From e257c8317be0e8810a8b3afa90922f868f98f6b3 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 9 Mar 2022 19:58:44 +0100 Subject: [PATCH 37/64] Optimize copying during join --- src/kiss.erl | 15 ++++++++++++--- src/kiss_join.erl | 10 ++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/kiss.erl b/src/kiss.erl index f18e7c28..04d82e8f 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -18,7 +18,7 @@ -behaviour(gen_server). -export([start/2, stop/1, insert/2, delete/2, delete_many/2]). --export([dump/1, remote_dump/1, send_dump_to_remote_node/3]). +-export([dump/1, remote_dump/1, send_dump_to_remote_node/3, table_name/1]). -export([other_nodes/1, other_pids/1]). -export([pause/1, unpause/1, sync/1, ping/1]). -export([info/1]). @@ -65,6 +65,11 @@ dump(Tab) -> remote_dump(Server) -> short_call(Server, remote_dump). +table_name(Tab) when is_atom(Tab) -> + Tab; +table_name(Server) -> + short_call(Server, table_name). + send_dump_to_remote_node(RemotePid, NewPids, OurDump) -> Msg = {send_dump_to_remote_node, NewPids, OurDump}, F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, @@ -166,8 +171,12 @@ handle_call(sync, _From, State = #{other_servers := Servers}) -> {reply, ok, State}; handle_call(ping, _From, State) -> {reply, ping, State}; -handle_call(remote_dump, _From, State = #{tab := Tab}) -> - {reply, {ok, dump(Tab)}, State}; +handle_call(table_name, _From, State = #{tab := Tab}) -> + {reply, {ok, Tab}, State}; +handle_call(remote_dump, From, State = #{tab := Tab}) -> + %% Do not block the main process (also reduces GC) + proc_lib:spawn_link(fun() -> gen_server:reply(From, {ok, dump(Tab)}) end), + {noreply, State}; handle_call({send_dump_to_remote_node, NewPids, Dump}, _From, State) -> handle_send_dump_to_remote_node(NewPids, Dump, State); handle_call(pause, _From = {FromPid, _}, State) -> diff --git a/src/kiss_join.erl b/src/kiss_join.erl index 7bbb31aa..c5585912 100644 --- a/src/kiss_join.erl +++ b/src/kiss_join.erl @@ -50,11 +50,17 @@ join2(_Info, LocalPid, RemotePid) -> try kiss:sync(LocalPid), kiss:sync(RemotePid), - {ok, LocalDump} = kiss:remote_dump(LocalPid), - {ok, RemoteDump} = kiss:remote_dump(RemotePid), + {ok, LocalDump} = remote_or_local_dump(LocalPid), + {ok, RemoteDump} = remote_or_local_dump(RemotePid), [kiss:send_dump_to_remote_node(Pid, LocPids, LocalDump) || Pid <- RemPids], [kiss:send_dump_to_remote_node(Pid, RemPids, RemoteDump) || Pid <- LocPids], ok after [kiss:unpause(Pid) || Pid <- AllPids] end. + +remote_or_local_dump(Pid) when node(Pid) =:= node() -> + {ok, Tab} = kiss:table_name(Pid), + {ok, kiss:dump(Tab)}; %% Reduce copying +remote_or_local_dump(Pid) -> + kiss:remote_dump(Pid). %% We actually need to ask the remote process From f1abf1b6782a658ba106293fb8633a258d0ceb1e Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 9 Mar 2022 20:21:52 +0100 Subject: [PATCH 38/64] Do joining in a new process --- src/kiss.erl | 2 +- src/kiss_clean.erl | 27 +++++++++++++++++++++++++++ src/kiss_join.erl | 3 ++- 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 src/kiss_clean.erl diff --git a/src/kiss.erl b/src/kiss.erl index 04d82e8f..08e3759f 100644 --- a/src/kiss.erl +++ b/src/kiss.erl @@ -174,7 +174,7 @@ handle_call(ping, _From, State) -> handle_call(table_name, _From, State = #{tab := Tab}) -> {reply, {ok, Tab}, State}; handle_call(remote_dump, From, State = #{tab := Tab}) -> - %% Do not block the main process (also reduces GC) + %% Do not block the main process (also reduces GC of the main process) proc_lib:spawn_link(fun() -> gen_server:reply(From, {ok, dump(Tab)}) end), {noreply, State}; handle_call({send_dump_to_remote_node, NewPids, Dump}, _From, State) -> diff --git a/src/kiss_clean.erl b/src/kiss_clean.erl new file mode 100644 index 00000000..eac546fc --- /dev/null +++ b/src/kiss_clean.erl @@ -0,0 +1,27 @@ +-module(kiss_clean). +-export([blocking/1]). + +-include_lib("kernel/include/logger.hrl"). + +%% Spawn a new process to do some memory-intensive task +%% This allows to reduce GC on the parent process +%% Wait for function to finish +%% Handles errors +blocking(F) -> + Pid = self(), + Ref = make_ref(), + proc_lib:spawn_link(fun() -> + Res = try + F() + catch Class:Reason:Stacktrace -> + ?LOG_ERROR(#{what => blocking_call_failed, + class => Class, reason => Reason, + stacktrace => Stacktrace}), + {error, {Class, Reason, Stacktrace}} + end, + Pid ! {result, Ref, Res} + end), + receive + {result, Ref, Res} -> + Res + end. diff --git a/src/kiss_join.erl b/src/kiss_join.erl index c5585912..67fc34d3 100644 --- a/src/kiss_join.erl +++ b/src/kiss_join.erl @@ -26,7 +26,8 @@ join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> %% Getting the lock could take really long time in case nodes are %% overloaded or joining is already in progress on another node ?LOG_INFO(Info#{what => join_got_lock, after_time_ms => Diff}), - join2(Info, LocalPid, RemotePid) + %% Do joining in a separate process to reduce GC + kiss_clean:blocking(fun() -> join2(Info, LocalPid, RemotePid) end) end, LockRequest = {LockKey, self()}, %% Just lock all nodes, no magic here :) From 81185a26ecdfb02e2371db93bc4ca11de8e6ab8e Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 9 Mar 2022 20:39:30 +0100 Subject: [PATCH 39/64] Put try..catch logic into kiss_safety --- src/kiss_clean.erl | 9 +-------- src/kiss_join.erl | 11 ++++++++--- src/kiss_long.erl | 9 ++++----- src/kiss_safety.erl | 11 +++++++++++ 4 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 src/kiss_safety.erl diff --git a/src/kiss_clean.erl b/src/kiss_clean.erl index eac546fc..d2eaeb20 100644 --- a/src/kiss_clean.erl +++ b/src/kiss_clean.erl @@ -11,14 +11,7 @@ blocking(F) -> Pid = self(), Ref = make_ref(), proc_lib:spawn_link(fun() -> - Res = try - F() - catch Class:Reason:Stacktrace -> - ?LOG_ERROR(#{what => blocking_call_failed, - class => Class, reason => Reason, - stacktrace => Stacktrace}), - {error, {Class, Reason, Stacktrace}} - end, + Res = kiss_safety:run(#{what => blocking_call_failed}, F), Pid ! {result, Ref, Res} end), receive diff --git a/src/kiss_join.erl b/src/kiss_join.erl index 67fc34d3..df4826e6 100644 --- a/src/kiss_join.erl +++ b/src/kiss_join.erl @@ -6,15 +6,20 @@ %% Writes from other nodes would wait for join completion. %% LockKey should be the same on all nodes. join(LockKey, Info, LocalPid, RemotePid) when is_pid(LocalPid), is_pid(RemotePid) -> - Info2 = Info#{local_pid => LocalPid, remote_pid => RemotePid, remote_node => node(RemotePid)}, + Info2 = Info#{local_pid => LocalPid, + remote_pid => RemotePid, remote_node => node(RemotePid)}, + F = fun() -> join1(LockKey, Info2, LocalPid, RemotePid) end, + kiss_safety:run(Info2#{what => join_failed}, F). + +join1(LockKey, Info, LocalPid, RemotePid) -> OtherPids = kiss:other_pids(LocalPid), case lists:member(RemotePid, OtherPids) of true -> {error, already_joined}; false -> Start = os:timestamp(), - F = fun() -> join_loop(LockKey, Info2, LocalPid, RemotePid, Start) end, - kiss_long:run(Info2#{task => join}, F) + F = fun() -> join_loop(LockKey, Info, LocalPid, RemotePid, Start) end, + kiss_long:run(Info#{task => join}, F) end. join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> diff --git a/src/kiss_long.erl b/src/kiss_long.erl index 5ca56cd3..588fd12a 100644 --- a/src/kiss_long.erl +++ b/src/kiss_long.erl @@ -16,11 +16,10 @@ run(Info, Fun, Catch) -> ?LOG_INFO(Info#{what => long_task_started}), Pid = spawn_mon(Info, Parent, Start), try - Fun() - catch Class:Reason:Stacktrace when Catch -> - ?LOG_INFO(Info#{what => long_task_failed, class => Class, - reason => Reason, stacktrace => Stacktrace}), - {error, {Class, Reason, Stacktrace}} + case Catch of + true -> kiss_safety:run(Info#{what => long_task_failed}, Fun); + false -> Fun() + end after Diff = diff(Start), ?LOG_INFO(Info#{what => long_task_finished, time_ms => Diff}), diff --git a/src/kiss_safety.erl b/src/kiss_safety.erl new file mode 100644 index 00000000..07f73d36 --- /dev/null +++ b/src/kiss_safety.erl @@ -0,0 +1,11 @@ +-module(kiss_safety). +-export([run/2]). +-include_lib("kernel/include/logger.hrl"). + +run(Info, Fun) -> + try + Fun() + catch Class:Reason:Stacktrace -> + ?LOG_ERROR(Info#{class => Class, reason => Reason, stacktrace => Stacktrace}), + {error, {Class, Reason, Stacktrace}} + end. From dc13d7812d6c3508e6dc4c38b13ca5d71b2e97fe Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 13 Apr 2022 10:37:31 +0200 Subject: [PATCH 40/64] Rename kiss to cets --- src/{kiss.app.src => cets.app.src} | 6 +- src/{kiss.erl => cets.erl} | 33 ++++--- src/{kiss_clean.erl => cets_clean.erl} | 4 +- ...{kiss_discovery.erl => cets_discovery.erl} | 8 +- ...overy_file.erl => cets_discovery_file.erl} | 4 +- src/{kiss_join.erl => cets_join.erl} | 32 +++--- src/{kiss_long.erl => cets_long.erl} | 4 +- ...s_mon_cleaner.erl => cets_mon_cleaner.erl} | 2 +- src/{kiss_safety.erl => cets_safety.erl} | 2 +- test/{kiss_SUITE.erl => cets_SUITE.erl} | 98 +++++++++---------- 10 files changed, 99 insertions(+), 94 deletions(-) rename src/{kiss.app.src => cets.app.src} (53%) rename src/{kiss.erl => cets.erl} (93%) rename src/{kiss_clean.erl => cets_clean.erl} (83%) rename src/{kiss_discovery.erl => cets_discovery.erl} (95%) rename src/{kiss_discovery_file.erl => cets_discovery_file.erl} (93%) rename src/{kiss_join.erl => cets_join.erl} (74%) rename src/{kiss_long.erl => cets_long.erl} (93%) rename src/{kiss_mon_cleaner.erl => cets_mon_cleaner.erl} (98%) rename src/{kiss_safety.erl => cets_safety.erl} (92%) rename test/{kiss_SUITE.erl => cets_SUITE.erl} (72%) diff --git a/src/kiss.app.src b/src/cets.app.src similarity index 53% rename from src/kiss.app.src rename to src/cets.app.src index f0339234..fced7441 100644 --- a/src/kiss.app.src +++ b/src/cets.app.src @@ -1,9 +1,9 @@ -{application, kiss, - [{description, "Simple Base"}, +{application, cets, + [{description, "Clustered Erlang Term Storage"}, {vsn, "0.1"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib]}, {env, []} -% {mod, {kiss_app, []} +% {mod, {cets_app, []} ]}. diff --git a/src/kiss.erl b/src/cets.erl similarity index 93% rename from src/kiss.erl rename to src/cets.erl index 08e3759f..9f56ade9 100644 --- a/src/kiss.erl +++ b/src/cets.erl @@ -14,7 +14,7 @@ %% - Add writer pid() or writer node() as a key. And do a proper cleanups using handle_down. %% (the data could still get overwritten though if a node joins back way too quick %% and cleaning is done outside of handle_down) --module(kiss). +-module(cets). -behaviour(gen_server). -export([start/2, stop/1, insert/2, delete/2, delete_many/2]). @@ -75,7 +75,7 @@ send_dump_to_remote_node(RemotePid, NewPids, OurDump) -> F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, Info = #{task => send_dump_to_remote_node, remote_pid => RemotePid, count => length(OurDump)}, - kiss_long:run_safely(Info, F). + cets_long:run_safely(Info, F). %% Only the node that owns the data could update/remove the data. %% Ideally Key should contain inserter node info (for cleaning). @@ -123,7 +123,7 @@ other_nodes(Server) -> -spec other_pids(server()) -> [pid()]. other_pids(Server) -> - servers_to_pids(other_servers(Server)). + other_servers(Server). -spec pause(server()) -> term(). pause(Server) -> @@ -153,7 +153,7 @@ init({Tab, Opts}) -> MonTab = list_to_atom(atom_to_list(Tab) ++ "_mon"), ets:new(Tab, [ordered_set, named_table, public]), ets:new(MonTab, [public, named_table]), - kiss_mon_cleaner:start_link(MonTab, MonTab), + cets_mon_cleaner:start_link(MonTab, MonTab), {ok, #{tab => Tab, mon_tab => MonTab, other_servers => [], opts => Opts, backlog => [], paused => false, pause_monitor => undefined}}. @@ -167,7 +167,7 @@ handle_call({delete, Keys}, From, State = #{paused := false}) -> handle_call(other_servers, _From, State = #{other_servers := Servers}) -> {reply, Servers, State}; handle_call(sync, _From, State = #{other_servers := Servers}) -> - [ping(Pid) || Pid <- servers_to_pids(Servers)], + [ping(Pid) || Pid <- Servers], {reply, ok, State}; handle_call(ping, _From, State) -> {reply, ping, State}; @@ -187,7 +187,9 @@ handle_call(unpause, _From, State) -> handle_call(get_info, _From, State) -> handle_get_info(State); handle_call(Msg, From, State = #{paused := true, backlog := Backlog}) -> - case should_backlogged(Msg) of + %% Backlog is a list of pending operation, when our server is paused. + %% The list would be applied, once our server is unpaused. + case should_backlog(Msg) of true -> {noreply, State#{backlog := [{Msg, From} | Backlog]}}; false -> @@ -259,6 +261,7 @@ notify_remote_down_loop(RemotePid, [{Mon, Pid} | List]) -> notify_remote_down_loop(_RemotePid, []) -> ok. +%% Merge two lists of pids, create the missing monitors. add_servers(Pids, Servers) -> lists:sort(add_servers2(Pids, Servers) ++ Servers). @@ -285,9 +288,6 @@ ets_delete_keys(Tab, [Key | Keys]) -> ets_delete_keys(_Tab, []) -> ok. -servers_to_pids(Servers) -> - [Pid || Pid <- Servers]. - has_remote_pid(RemotePid, Servers) -> lists:member(RemotePid, Servers). @@ -325,6 +325,8 @@ replicate2([RemotePid | Servers], Msg) -> replicate2([], _Msg) -> ok. +%% Wait for response from the remote nodes that the operation is completed. +%% remote_down is sent by the local server, if the remote server is down. wait_for_updated({Mon, Servers, MonTab}) -> try wait_for_updated2(Mon, Servers) @@ -339,6 +341,9 @@ wait_for_updated2(Mon, Servers) -> {updated, Mon, Pid} -> Servers2 = lists:delete(Pid, Servers), wait_for_updated2(Mon, Servers2); + %% What happens if the main server dies? + %% Technically, we could add a monitor, so we detect that case. + %% But if the server dies, we should stop the node anyway. {remote_down, Mon, Pid} -> Servers2 = lists:delete(Pid, Servers), wait_for_updated2(Mon, Servers2) @@ -356,7 +361,7 @@ short_call(RemotePid, Msg) when is_pid(RemotePid) -> F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, Info = #{task => Msg, remote_pid => RemotePid, remote_node => node(RemotePid)}, - kiss_long:run_safely(Info, F); + cets_long:run_safely(Info, F); short_call(Name, Msg) when is_atom(Name) -> short_call(whereis(Name), Msg). @@ -383,11 +388,11 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> FF = fun() -> F(#{remote_pid => RemotePid, table => Tab}) end, Info = #{task => call_user_handle_down, table => Tab, remote_pid => RemotePid, remote_node => node(RemotePid)}, - kiss_long:run_safely(Info, FF); + cets_long:run_safely(Info, FF); _ -> ok end. -should_backlogged({insert, _}) -> true; -should_backlogged({delete, _}) -> true; -should_backlogged(_) -> false. +should_backlog({insert, _}) -> true; +should_backlog({delete, _}) -> true; +should_backlog(_) -> false. diff --git a/src/kiss_clean.erl b/src/cets_clean.erl similarity index 83% rename from src/kiss_clean.erl rename to src/cets_clean.erl index d2eaeb20..ec4a3a02 100644 --- a/src/kiss_clean.erl +++ b/src/cets_clean.erl @@ -1,4 +1,4 @@ --module(kiss_clean). +-module(cets_clean). -export([blocking/1]). -include_lib("kernel/include/logger.hrl"). @@ -11,7 +11,7 @@ blocking(F) -> Pid = self(), Ref = make_ref(), proc_lib:spawn_link(fun() -> - Res = kiss_safety:run(#{what => blocking_call_failed}, F), + Res = cets_safety:run(#{what => blocking_call_failed}, F), Pid ! {result, Ref, Res} end), receive diff --git a/src/kiss_discovery.erl b/src/cets_discovery.erl similarity index 95% rename from src/kiss_discovery.erl rename to src/cets_discovery.erl index 069591b5..216556c3 100644 --- a/src/kiss_discovery.erl +++ b/src/cets_discovery.erl @@ -1,5 +1,5 @@ %% Joins table together when a new node appears --module(kiss_discovery). +-module(cets_discovery). -behaviour(gen_server). -export([start/1, start_link/1, add_table/2, info/1]). @@ -47,11 +47,11 @@ get_tables(Server) -> info(Server) -> {ok, Tables} = get_tables(Server), - [kiss:info(Tab) || Tab <- Tables]. + [cets:info(Tab) || Tab <- Tables]. -spec init(term()) -> {ok, state()}. init(Opts) -> - Mod = maps:get(backend_module, Opts, kiss_discovery_file), + Mod = maps:get(backend_module, Opts, cets_discovery_file), self() ! check, Tables = maps:get(tables, Opts, []), BackendState = Mod:init(Opts), @@ -123,7 +123,7 @@ do_join(Tab, Node) -> %% That would trigger autoconnect for the first time case rpc:call(Node, erlang, whereis, [Tab]) of Pid when is_pid(Pid), is_pid(LocalPid) -> - Result = kiss_join:join(kiss_discovery, #{table => Tab}, LocalPid, Pid), + Result = cets_join:join(cets_discovery, #{table => Tab}, LocalPid, Pid), #{what => join_result, result => Result, node => Node, table => Tab}; Other -> #{what => pid_not_found, reason => Other, node => Node, table => Tab} diff --git a/src/kiss_discovery_file.erl b/src/cets_discovery_file.erl similarity index 93% rename from src/kiss_discovery_file.erl rename to src/cets_discovery_file.erl index 2c461763..cdad52f1 100644 --- a/src/kiss_discovery_file.erl +++ b/src/cets_discovery_file.erl @@ -4,8 +4,8 @@ %% - DNS does not allow to list subdomains %% So, we use a file with nodes to connect as a discovery mechanism %% (so, you can hardcode nodes or use your method of filling it) --module(kiss_discovery_file). --behaviour(kiss_discovery). +-module(cets_discovery_file). +-behaviour(cets_discovery). -export([init/1, get_nodes/1]). -include_lib("kernel/include/logger.hrl"). diff --git a/src/kiss_join.erl b/src/cets_join.erl similarity index 74% rename from src/kiss_join.erl rename to src/cets_join.erl index df4826e6..0c256cdd 100644 --- a/src/kiss_join.erl +++ b/src/cets_join.erl @@ -1,4 +1,4 @@ --module(kiss_join). +-module(cets_join). -export([join/4]). -include_lib("kernel/include/logger.hrl"). @@ -9,17 +9,17 @@ join(LockKey, Info, LocalPid, RemotePid) when is_pid(LocalPid), is_pid(RemotePid Info2 = Info#{local_pid => LocalPid, remote_pid => RemotePid, remote_node => node(RemotePid)}, F = fun() -> join1(LockKey, Info2, LocalPid, RemotePid) end, - kiss_safety:run(Info2#{what => join_failed}, F). + cets_safety:run(Info2#{what => join_failed}, F). join1(LockKey, Info, LocalPid, RemotePid) -> - OtherPids = kiss:other_pids(LocalPid), + OtherPids = cets:other_pids(LocalPid), case lists:member(RemotePid, OtherPids) of true -> {error, already_joined}; false -> Start = os:timestamp(), F = fun() -> join_loop(LockKey, Info, LocalPid, RemotePid, Start) end, - kiss_long:run(Info#{task => join}, F) + cets_long:run(Info#{task => join}, F) end. join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> @@ -32,7 +32,7 @@ join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> %% overloaded or joining is already in progress on another node ?LOG_INFO(Info#{what => join_got_lock, after_time_ms => Diff}), %% Do joining in a separate process to reduce GC - kiss_clean:blocking(fun() -> join2(Info, LocalPid, RemotePid) end) + cets_clean:blocking(fun() -> join2(Info, LocalPid, RemotePid) end) end, LockRequest = {LockKey, self()}, %% Just lock all nodes, no magic here :) @@ -47,26 +47,26 @@ join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> end. join2(_Info, LocalPid, RemotePid) -> - LocalOtherPids = kiss:other_pids(LocalPid), - RemoteOtherPids = kiss:other_pids(RemotePid), + LocalOtherPids = cets:other_pids(LocalPid), + RemoteOtherPids = cets:other_pids(RemotePid), LocPids = [LocalPid | LocalOtherPids], RemPids = [RemotePid | RemoteOtherPids], AllPids = LocPids ++ RemPids, - [kiss:pause(Pid) || Pid <- AllPids], + [cets:pause(Pid) || Pid <- AllPids], try - kiss:sync(LocalPid), - kiss:sync(RemotePid), + cets:sync(LocalPid), + cets:sync(RemotePid), {ok, LocalDump} = remote_or_local_dump(LocalPid), {ok, RemoteDump} = remote_or_local_dump(RemotePid), - [kiss:send_dump_to_remote_node(Pid, LocPids, LocalDump) || Pid <- RemPids], - [kiss:send_dump_to_remote_node(Pid, RemPids, RemoteDump) || Pid <- LocPids], + [cets:send_dump_to_remote_node(Pid, LocPids, LocalDump) || Pid <- RemPids], + [cets:send_dump_to_remote_node(Pid, RemPids, RemoteDump) || Pid <- LocPids], ok after - [kiss:unpause(Pid) || Pid <- AllPids] + [cets:unpause(Pid) || Pid <- AllPids] end. remote_or_local_dump(Pid) when node(Pid) =:= node() -> - {ok, Tab} = kiss:table_name(Pid), - {ok, kiss:dump(Tab)}; %% Reduce copying + {ok, Tab} = cets:table_name(Pid), + {ok, cets:dump(Tab)}; %% Reduce copying remote_or_local_dump(Pid) -> - kiss:remote_dump(Pid). %% We actually need to ask the remote process + cets:remote_dump(Pid). %% We actually need to ask the remote process diff --git a/src/kiss_long.erl b/src/cets_long.erl similarity index 93% rename from src/kiss_long.erl rename to src/cets_long.erl index 588fd12a..b7fb3ba5 100644 --- a/src/kiss_long.erl +++ b/src/cets_long.erl @@ -1,4 +1,4 @@ --module(kiss_long). +-module(cets_long). -export([run/2]). -export([run_safely/2]). @@ -17,7 +17,7 @@ run(Info, Fun, Catch) -> Pid = spawn_mon(Info, Parent, Start), try case Catch of - true -> kiss_safety:run(Info#{what => long_task_failed}, Fun); + true -> cets_safety:run(Info#{what => long_task_failed}, Fun); false -> Fun() end after diff --git a/src/kiss_mon_cleaner.erl b/src/cets_mon_cleaner.erl similarity index 98% rename from src/kiss_mon_cleaner.erl rename to src/cets_mon_cleaner.erl index a79a250f..b9f03017 100644 --- a/src/kiss_mon_cleaner.erl +++ b/src/cets_mon_cleaner.erl @@ -3,7 +3,7 @@ %% Unless the caller process crashes. %% This server removes such entries from the MonTab. %% We don't expect the MonTab to be extremely big, so this check should be quick. --module(kiss_mon_cleaner). +-module(cets_mon_cleaner). -behaviour(gen_server). -export([start_link/2]). diff --git a/src/kiss_safety.erl b/src/cets_safety.erl similarity index 92% rename from src/kiss_safety.erl rename to src/cets_safety.erl index 07f73d36..f3c07ab6 100644 --- a/src/kiss_safety.erl +++ b/src/cets_safety.erl @@ -1,4 +1,4 @@ --module(kiss_safety). +-module(cets_safety). -export([run/2]). -include_lib("kernel/include/logger.hrl"). diff --git a/test/kiss_SUITE.erl b/test/cets_SUITE.erl similarity index 72% rename from test/kiss_SUITE.erl rename to test/cets_SUITE.erl index 6d0935ea..554c5884 100644 --- a/test/kiss_SUITE.erl +++ b/test/cets_SUITE.erl @@ -1,4 +1,4 @@ --module(kiss_SUITE). +-module(cets_SUITE). -include_lib("common_test/include/ct.hrl"). -compile([export_all, nowarn_export_all]). @@ -94,32 +94,32 @@ test_multinode_auto_discovery(Config) -> ct:pal("Dir ~p", [Dir]), FileName = filename:join(Dir, "disco.txt"), ok = file:write_file(FileName, io_lib:format("~s~n~s~n", [Node1, Node2])), - {ok, Disco} = kiss_discovery:start(#{tables => [Tab], disco_file => FileName}), + {ok, Disco} = cets_discovery:start(#{tables => [Tab], disco_file => FileName}), %% Waits for the first check sys:get_state(Disco), [Node2] = other_nodes(Node1, Tab), [#{memory := _, nodes := [Node1, Node2], size := 0, table := tab2}] - = kiss_discovery:info(Disco), + = cets_discovery:info(Disco), ok. test_locally(_Config) -> - {ok, Pid1} = kiss:start(t1, #{}), - {ok, Pid2} = kiss:start(t2, #{}), - ok = kiss_join:join(lock1, #{table => [t1, t2]}, Pid1, Pid2), - kiss:insert(t1, {1}), - kiss:insert(t1, {1}), - kiss:insert(t2, {2}), - D = kiss:dump(t1), - D = kiss:dump(t2). + {ok, Pid1} = cets:start(t1, #{}), + {ok, Pid2} = cets:start(t2, #{}), + ok = cets_join:join(lock1, #{table => [t1, t2]}, Pid1, Pid2), + cets:insert(t1, {1}), + cets:insert(t1, {1}), + cets:insert(t2, {2}), + D = cets:dump(t1), + D = cets:dump(t2). handle_down_is_called(_Config) -> Parent = self(), DownFn = fun(#{remote_pid := _RemotePid, table := _Tab}) -> Parent ! down_called end, - {ok, Pid1} = kiss:start(d1, #{handle_down => DownFn}), - {ok, Pid2} = kiss:start(d2, #{}), - ok = kiss_join:join(lock1, #{table => [d1, d2]}, Pid1, Pid2), + {ok, Pid1} = cets:start(d1, #{handle_down => DownFn}), + {ok, Pid2} = cets:start(d2, #{}), + ok = cets_join:join(lock1, #{table => [d1, d2]}, Pid1, Pid2), exit(Pid2, oops), receive down_called -> ok @@ -128,48 +128,48 @@ handle_down_is_called(_Config) -> events_are_applied_in_the_correct_order_after_unpause(_Config) -> T = t4, - {ok, Pid} = kiss:start(T, #{}), - ok = kiss:pause(Pid), - R1 = kiss:insert_request(T, {1}), - R2 = kiss:delete_request(T, 1), - kiss:delete_request(T, 2), - kiss:insert_request(T, {2}), - kiss:insert_request(T, {3}), - kiss:insert_request(T, {4}), - kiss:insert_request(T, {5}), - R3 = kiss:insert_request(T, [{6}, {7}]), - R4 = kiss:delete_many_request(T, [5, 4]), - [] = lists:sort(kiss:dump(T)), - ok = kiss:unpause(Pid), - [ok = kiss:wait_response(R, 5000) || R <- [R1, R2, R3, R4]], - [{2}, {3}, {6}, {7}] = lists:sort(kiss:dump(T)). + {ok, Pid} = cets:start(T, #{}), + ok = cets:pause(Pid), + R1 = cets:insert_request(T, {1}), + R2 = cets:delete_request(T, 1), + cets:delete_request(T, 2), + cets:insert_request(T, {2}), + cets:insert_request(T, {3}), + cets:insert_request(T, {4}), + cets:insert_request(T, {5}), + R3 = cets:insert_request(T, [{6}, {7}]), + R4 = cets:delete_many_request(T, [5, 4]), + [] = lists:sort(cets:dump(T)), + ok = cets:unpause(Pid), + [ok = cets:wait_response(R, 5000) || R <- [R1, R2, R3, R4]], + [{2}, {3}, {6}, {7}] = lists:sort(cets:dump(T)). write_returns_if_remote_server_crashes(_Config) -> - {ok, Pid1} = kiss:start(c1, #{}), - {ok, Pid2} = kiss:start(c2, #{}), - ok = kiss_join:join(lock1, #{table => [c1, c2]}, Pid1, Pid2), + {ok, Pid1} = cets:start(c1, #{}), + {ok, Pid2} = cets:start(c2, #{}), + ok = cets_join:join(lock1, #{table => [c1, c2]}, Pid1, Pid2), sys:suspend(Pid2), - R = kiss:insert_request(c1, {1}), + R = cets:insert_request(c1, {1}), exit(Pid2, oops), - ok = kiss:wait_response(R, 5000). + ok = cets:wait_response(R, 5000). mon_cleaner_works(_Config) -> - {ok, Pid1} = kiss:start(c3, #{}), + {ok, Pid1} = cets:start(c3, #{}), %% Suspend, so to avoid unexpected check sys:suspend(c3_mon), %% Two cases to check: an alive process and a dead process - R = kiss:insert_request(c3, {2}), + R = cets:insert_request(c3, {2}), %% Ensure insert_request reaches the server - kiss:ping(Pid1), + cets:ping(Pid1), %% There is one monitor [_] = ets:tab2list(c3_mon), - {Pid, Mon} = spawn_monitor(fun() -> kiss:insert_request(c3, {1}) end), + {Pid, Mon} = spawn_monitor(fun() -> cets:insert_request(c3, {1}) end), receive {'DOWN', Mon, process, Pid, _Reason} -> ok after 5000 -> ct:fail(timeout) end, %% Ensure insert_request reaches the server - kiss:ping(Pid1), + cets:ping(Pid1), %% There are two monitors [_, _] = ets:tab2list(c3_mon), %% Force check @@ -180,33 +180,33 @@ mon_cleaner_works(_Config) -> %% A monitor for a dead process is removed [_] = ets:tab2list(c3_mon), %% The monitor is finally removed once wait_response returns - ok = kiss:wait_response(R, 5000), + ok = cets:wait_response(R, 5000), [] = ets:tab2list(c3_mon). sync_using_name_works(_Config) -> - {ok, _Pid1} = kiss:start(c4, #{}), - kiss:sync(c4). + {ok, _Pid1} = cets:start(c4, #{}), + cets:sync(c4). start(Node, Tab) -> - rpc(Node, kiss, start, [Tab, #{}]). + rpc(Node, cets, start, [Tab, #{}]). insert(Node, Tab, Rec) -> - rpc(Node, kiss, insert, [Tab, Rec]). + rpc(Node, cets, insert, [Tab, Rec]). delete(Node, Tab, Key) -> - rpc(Node, kiss, delete, [Tab, Key]). + rpc(Node, cets, delete, [Tab, Key]). delete_many(Node, Tab, Keys) -> - rpc(Node, kiss, delete_many, [Tab, Keys]). + rpc(Node, cets, delete_many, [Tab, Keys]). dump(Node, Tab) -> - rpc(Node, kiss, dump, [Tab]). + rpc(Node, cets, dump, [Tab]). other_nodes(Node, Tab) -> - rpc(Node, kiss, other_nodes, [Tab]). + rpc(Node, cets, other_nodes, [Tab]). join(Node1, Tab, Pid1, Pid2) -> - rpc(Node1, kiss_join, join, [lock1, #{table => Tab}, Pid1, Pid2]). + rpc(Node1, cets_join, join, [lock1, #{table => Tab}, Pid1, Pid2]). rpc(Node, M, F, Args) -> case rpc:call(Node, M, F, Args) of From aa886485e24a8cd170f2cd5a94fb7ad453cf5e5e Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 29 Apr 2022 16:22:07 +0200 Subject: [PATCH 41/64] Use erlang:system_time api --- src/cets.erl | 2 -- src/cets_join.erl | 4 ++-- src/cets_long.erl | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/cets.erl b/src/cets.erl index 9f56ade9..aab94e24 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -342,8 +342,6 @@ wait_for_updated2(Mon, Servers) -> Servers2 = lists:delete(Pid, Servers), wait_for_updated2(Mon, Servers2); %% What happens if the main server dies? - %% Technically, we could add a monitor, so we detect that case. - %% But if the server dies, we should stop the node anyway. {remote_down, Mon, Pid} -> Servers2 = lists:delete(Pid, Servers), wait_for_updated2(Mon, Servers2) diff --git a/src/cets_join.erl b/src/cets_join.erl index 0c256cdd..29e64fa6 100644 --- a/src/cets_join.erl +++ b/src/cets_join.erl @@ -17,7 +17,7 @@ join1(LockKey, Info, LocalPid, RemotePid) -> true -> {error, already_joined}; false -> - Start = os:timestamp(), + Start = erlang:system_time(millisecond), F = fun() -> join_loop(LockKey, Info, LocalPid, RemotePid, Start) end, cets_long:run(Info#{task => join}, F) end. @@ -27,7 +27,7 @@ join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> %% - for performance reasons, we don't want to cause too much load for active nodes %% - to avoid deadlocks, because joining does gen_server calls F = fun() -> - Diff = timer:now_diff(os:timestamp(), Start) div 1000, + Diff = erlang:system_time(millisecond) - Start, %% Getting the lock could take really long time in case nodes are %% overloaded or joining is already in progress on another node ?LOG_INFO(Info#{what => join_got_lock, after_time_ms => Diff}), diff --git a/src/cets_long.erl b/src/cets_long.erl index b7fb3ba5..aeb94002 100644 --- a/src/cets_long.erl +++ b/src/cets_long.erl @@ -12,7 +12,7 @@ run(Info, Fun) -> run(Info, Fun, Catch) -> Parent = self(), - Start = os:timestamp(), + Start = erlang:system_time(millisecond), ?LOG_INFO(Info#{what => long_task_started}), Pid = spawn_mon(Info, Parent, Start), try @@ -46,4 +46,4 @@ monitor_loop(Mon, Info, Start) -> end. diff(Start) -> - timer:now_diff(os:timestamp(), Start) div 1000. + erlang:system_time(millisecond) - Start. From dffd76d53dfd46b2f9d81393dc7a39cbbb278ab5 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 29 Apr 2022 16:37:17 +0200 Subject: [PATCH 42/64] Remove cets_safety module --- src/cets_clean.erl | 20 -------------------- src/cets_join.erl | 7 +++++-- src/cets_long.erl | 30 +++++++++++++++++++++++++++--- src/cets_safety.erl | 11 ----------- 4 files changed, 32 insertions(+), 36 deletions(-) delete mode 100644 src/cets_clean.erl delete mode 100644 src/cets_safety.erl diff --git a/src/cets_clean.erl b/src/cets_clean.erl deleted file mode 100644 index ec4a3a02..00000000 --- a/src/cets_clean.erl +++ /dev/null @@ -1,20 +0,0 @@ --module(cets_clean). --export([blocking/1]). - --include_lib("kernel/include/logger.hrl"). - -%% Spawn a new process to do some memory-intensive task -%% This allows to reduce GC on the parent process -%% Wait for function to finish -%% Handles errors -blocking(F) -> - Pid = self(), - Ref = make_ref(), - proc_lib:spawn_link(fun() -> - Res = cets_safety:run(#{what => blocking_call_failed}, F), - Pid ! {result, Ref, Res} - end), - receive - {result, Ref, Res} -> - Res - end. diff --git a/src/cets_join.erl b/src/cets_join.erl index 29e64fa6..7ba9cb6d 100644 --- a/src/cets_join.erl +++ b/src/cets_join.erl @@ -9,7 +9,7 @@ join(LockKey, Info, LocalPid, RemotePid) when is_pid(LocalPid), is_pid(RemotePid Info2 = Info#{local_pid => LocalPid, remote_pid => RemotePid, remote_node => node(RemotePid)}, F = fun() -> join1(LockKey, Info2, LocalPid, RemotePid) end, - cets_safety:run(Info2#{what => join_failed}, F). + cets_long:run_safely(Info2#{what => join_failed}, F). join1(LockKey, Info, LocalPid, RemotePid) -> OtherPids = cets:other_pids(LocalPid), @@ -32,7 +32,7 @@ join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> %% overloaded or joining is already in progress on another node ?LOG_INFO(Info#{what => join_got_lock, after_time_ms => Diff}), %% Do joining in a separate process to reduce GC - cets_clean:blocking(fun() -> join2(Info, LocalPid, RemotePid) end) + cets_long:run_spawn(Info, fun() -> join2(Info, LocalPid, RemotePid) end) end, LockRequest = {LockKey, self()}, %% Just lock all nodes, no magic here :) @@ -53,6 +53,9 @@ join2(_Info, LocalPid, RemotePid) -> RemPids = [RemotePid | RemoteOtherPids], AllPids = LocPids ++ RemPids, [cets:pause(Pid) || Pid <- AllPids], + %% Merges data from two partitions together. + %% Each entry in the table is allowed to be updated by the node that owns + %% the key only, so merging is easy. try cets:sync(LocalPid), cets:sync(RemotePid), diff --git a/src/cets_long.erl b/src/cets_long.erl index aeb94002..17ee0eab 100644 --- a/src/cets_long.erl +++ b/src/cets_long.erl @@ -1,9 +1,25 @@ +%% Helper to log long running operations. -module(cets_long). --export([run/2]). --export([run_safely/2]). +-export([run_spawn/2, run/2, run_safely/2]). -include_lib("kernel/include/logger.hrl"). +%% Spawn a new process to do some memory-intensive task +%% This allows to reduce GC on the parent process +%% Wait for function to finish +%% Handles errors +run_spawn(Info, F) -> + Pid = self(), + Ref = make_ref(), + proc_lib:spawn_link(fun() -> + Res = cets_long:run_safely(Info, F), + Pid ! {result, Ref, Res} + end), + receive + {result, Ref, Res} -> + Res + end. + run_safely(Info, Fun) -> run(Info, Fun, true). @@ -17,7 +33,7 @@ run(Info, Fun, Catch) -> Pid = spawn_mon(Info, Parent, Start), try case Catch of - true -> cets_safety:run(Info#{what => long_task_failed}, Fun); + true -> just_run_safely(Info#{what => long_task_failed}, Fun); false -> Fun() end after @@ -47,3 +63,11 @@ monitor_loop(Mon, Info, Start) -> diff(Start) -> erlang:system_time(millisecond) - Start. + +just_run_safely(Info, Fun) -> + try + Fun() + catch Class:Reason:Stacktrace -> + ?LOG_ERROR(Info#{class => Class, reason => Reason, stacktrace => Stacktrace}), + {error, {Class, Reason, Stacktrace}} + end. diff --git a/src/cets_safety.erl b/src/cets_safety.erl deleted file mode 100644 index f3c07ab6..00000000 --- a/src/cets_safety.erl +++ /dev/null @@ -1,11 +0,0 @@ --module(cets_safety). --export([run/2]). --include_lib("kernel/include/logger.hrl"). - -run(Info, Fun) -> - try - Fun() - catch Class:Reason:Stacktrace -> - ?LOG_ERROR(Info#{class => Class, reason => Reason, stacktrace => Stacktrace}), - {error, {Class, Reason, Stacktrace}} - end. From 49dbddebc7a16ecebd864760fe3d68fc1e868413 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 29 Apr 2022 16:44:11 +0200 Subject: [PATCH 43/64] Avoid deadlocks in sync/1 --- src/cets.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/cets.erl b/src/cets.erl index aab94e24..558f8b3f 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -133,6 +133,7 @@ pause(Server) -> unpause(Server) -> short_call(Server, unpause). +%% Waits till all pending operations are applied. -spec sync(server()) -> term(). sync(Server) -> short_call(Server, sync). @@ -166,9 +167,13 @@ handle_call({delete, Keys}, From, State = #{paused := false}) -> handle_delete(Keys, From, State); handle_call(other_servers, _From, State = #{other_servers := Servers}) -> {reply, Servers, State}; -handle_call(sync, _From, State = #{other_servers := Servers}) -> - [ping(Pid) || Pid <- Servers], - {reply, ok, State}; +handle_call(sync, From, State = #{other_servers := Servers}) -> + %% Do spawn to avoid any possible deadlocks + proc_lib:spawn(fun() -> + [ping(Pid) || Pid <- Servers], + gen_server:reply(From, ok) + end), + {noreply, State}; handle_call(ping, _From, State) -> {reply, ping, State}; handle_call(table_name, _From, State = #{tab := Tab}) -> From 9219a4fef532bfeef9175f748f38712a508d7f6d Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 4 May 2022 16:12:00 +0200 Subject: [PATCH 44/64] Add Github CI Update readme --- .github/workflows/ci.yml | 24 +++++++++++++++++ README.md | 57 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..09cecdb7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,24 @@ +name: Erlang CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + + container: + image: erlang:23 + + steps: + - uses: actions/checkout@v3 + - name: Compile + run: rebar3 compile + - name: Run tests + run: rebar3 ct --sname=ct1 diff --git a/README.md b/README.md index 07b7f0d2..fdc2dd7b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,55 @@ -``` -./rebar3 ct --sname=ct -``` +# Cluster ETS +The project adds replication support for Erlang Term Storage (ETS). + +It allows to insert or delete objects into ETS tables across several Erlang nodes. + +The closest comparison is Mnesia with dirty operations. + +Some features are not supported: +- there is no schema +- there is no persistency +- there are no indexes +- there are no transactions (there are bulk inserts/deletes instead). + +# Merging logic + +When two database partitions are joined together, records from both partitions +are put together. So, each record would be present on each node. + +The idea that each node updates only records that it owns. The node name +should be inside an ETS key for this to work (or a pid). + +When some node is down, we remove all records that are owned by this node. +When a node reappears, the records are added back. + +# API + +The main module is cets. + +It exports functions: + +- `start(Tab, Opts)` - starts a new table manager. + There is one gen_server for each node for each table. +- `insert(Server, Rec)` - inserts a new object into a table. +- `insert_many(Server, Records)` - inserts several objects into a table. +- `delete(Server, Key)` - deletes an object from the table. +- `delete_many(Server, Keys)` - deletes several objects from the table. + +cets_join module contains the merging logic. + +cets_discovery module handles search of new nodes. + +It supports behaviours for different backends. + +It defines two callbacks: + +- `init/1` - inits the backend. +- `get_nodes/1` - gets a list of alive erlang nodes. + +Once new nodes are found, cets_discovery calls cets_join module to merge two +cluster partitions. + +The simplest cets_discovery backend is cets_discovery_file, which just reads +a file with a list of nodes on each line. This file could be populated by an +external program or by an admin. From 91a0c9250e34532f595ff7ab48123fab098709ed Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 19 Apr 2022 10:57:55 +0200 Subject: [PATCH 45/64] Add insert_many_request and insert_many API functions --- src/cets.erl | 16 +++++++++++++--- test/cets_SUITE.erl | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/cets.erl b/src/cets.erl index 558f8b3f..fc76c397 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -17,14 +17,15 @@ -module(cets). -behaviour(gen_server). --export([start/2, stop/1, insert/2, delete/2, delete_many/2]). +-export([start/2, stop/1, insert/2, insert_many/2, delete/2, delete_many/2]). -export([dump/1, remote_dump/1, send_dump_to_remote_node/3, table_name/1]). -export([other_nodes/1, other_pids/1]). -export([pause/1, unpause/1, sync/1, ping/1]). -export([info/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --export([insert_request/2, delete_request/2, delete_many_request/2, wait_response/2]). +-export([insert_request/2, insert_many_request/2, + delete_request/2, delete_many_request/2, wait_response/2]). -include_lib("kernel/include/logger.hrl"). @@ -80,10 +81,15 @@ send_dump_to_remote_node(RemotePid, NewPids, OurDump) -> %% Only the node that owns the data could update/remove the data. %% Ideally Key should contain inserter node info (for cleaning). -spec insert(server(), tuple()) -> ok. -insert(Server, Rec) -> +insert(Server, Rec) when is_tuple(Rec) -> {ok, Monitors} = gen_server:call(Server, {insert, Rec}), wait_for_updated(Monitors). +-spec insert_many(server(), list(tuple())) -> ok. +insert_many(Server, Records) when is_list(Records) -> + {ok, Monitors} = gen_server:call(Server, {insert, Records}), + wait_for_updated(Monitors). + -spec delete(server(), term()) -> ok. delete(Server, Key) -> delete_many(Server, [Key]). @@ -98,6 +104,10 @@ delete_many(Server, Keys) -> insert_request(Server, Rec) -> gen_server:send_request(Server, {insert, Rec}). +-spec insert_many_request(server(), [tuple()]) -> request_id(). +insert_many_request(Server, Records) -> + gen_server:send_request(Server, {insert, Records}). + -spec delete_request(server(), term()) -> request_id(). delete_request(Tab, Key) -> delete_many_request(Tab, [Key]). diff --git a/test/cets_SUITE.erl b/test/cets_SUITE.erl index 554c5884..a33c5243 100644 --- a/test/cets_SUITE.erl +++ b/test/cets_SUITE.erl @@ -8,7 +8,8 @@ all() -> [test_multinode, node_list_is_correct, handle_down_is_called, events_are_applied_in_the_correct_order_after_unpause, write_returns_if_remote_server_crashes, - mon_cleaner_works, sync_using_name_works]. + mon_cleaner_works, sync_using_name_works, + insert_many_request]. init_per_suite(Config) -> Node2 = start_node(ct2), @@ -61,7 +62,7 @@ test_multinode(Config) -> delete(Node4, Tab, a), Same([{b}, {c}, {d}, {f}]), %% Bulk operations are supported - insert(Node4, Tab, [{m}, {a}, {n}, {y}]), + insert_many(Node4, Tab, [{m}, {a}, {n}, {y}]), Same([{a}, {b}, {c}, {d}, {f}, {m}, {n}, {y}]), delete_many(Node4, Tab, [a,n]), Same([{b}, {c}, {d}, {f}, {m}, {y}]), @@ -187,12 +188,21 @@ sync_using_name_works(_Config) -> {ok, _Pid1} = cets:start(c4, #{}), cets:sync(c4). +insert_many_request(_Config) -> + {ok, Pid} = cets:start(c5, #{}), + R = cets:insert_many_request(Pid, [{a}, {b}]), + ok = cets:wait_response(R, 5000), + [{a}, {b}] = ets:tab2list(c5). + start(Node, Tab) -> rpc(Node, cets, start, [Tab, #{}]). insert(Node, Tab, Rec) -> rpc(Node, cets, insert, [Tab, Rec]). +insert_many(Node, Tab, Records) -> + rpc(Node, cets, insert_many, [Tab, Records]). + delete(Node, Tab, Key) -> rpc(Node, cets, delete, [Tab, Key]). From 91ff0b995851c34c8827635b3cbac752a3255584 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 4 May 2022 17:55:43 +0200 Subject: [PATCH 46/64] Allow multiple pauses --- src/cets.erl | 244 ++++++++++++++++++++++++-------------------- src/cets_join.erl | 4 +- test/cets_SUITE.erl | 4 +- 3 files changed, 136 insertions(+), 116 deletions(-) diff --git a/src/cets.erl b/src/cets.erl index fc76c397..8b553bab 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -20,7 +20,7 @@ -export([start/2, stop/1, insert/2, insert_many/2, delete/2, delete_many/2]). -export([dump/1, remote_dump/1, send_dump_to_remote_node/3, table_name/1]). -export([other_nodes/1, other_pids/1]). --export([pause/1, unpause/1, sync/1, ping/1]). +-export([pause/1, unpause/2, sync/1, ping/1]). -export([info/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). @@ -40,8 +40,7 @@ other_servers := [pid()], opts := map(), backlog := [backlog_entry()], - paused := false, - pause_monitor := undefined | reference()}. + pause_monitors := [reference()]}. %% API functions @@ -82,13 +81,11 @@ send_dump_to_remote_node(RemotePid, NewPids, OurDump) -> %% Ideally Key should contain inserter node info (for cleaning). -spec insert(server(), tuple()) -> ok. insert(Server, Rec) when is_tuple(Rec) -> - {ok, Monitors} = gen_server:call(Server, {insert, Rec}), - wait_for_updated(Monitors). + sync_operation(Server, {insert, Rec}). -spec insert_many(server(), list(tuple())) -> ok. insert_many(Server, Records) when is_list(Records) -> - {ok, Monitors} = gen_server:call(Server, {insert, Records}), - wait_for_updated(Monitors). + sync_operation(Server, {insert, Records}). -spec delete(server(), term()) -> ok. delete(Server, Key) -> @@ -97,16 +94,15 @@ delete(Server, Key) -> %% A separate function for multidelete (because key COULD be a list, so no confusion) -spec delete_many(server(), [term()]) -> ok. delete_many(Server, Keys) -> - {ok, WaitInfo} = gen_server:call(Server, {delete, Keys}), - wait_for_updated(WaitInfo). + sync_operation(Server, {delete, Keys}). -spec insert_request(server(), tuple()) -> request_id(). insert_request(Server, Rec) -> - gen_server:send_request(Server, {insert, Rec}). + async_operation(Server, {insert, Rec}). -spec insert_many_request(server(), [tuple()]) -> request_id(). insert_many_request(Server, Records) -> - gen_server:send_request(Server, {insert, Records}). + async_operation(Server, {insert, Records}). -spec delete_request(server(), term()) -> request_id(). delete_request(Tab, Key) -> @@ -114,16 +110,7 @@ delete_request(Tab, Key) -> -spec delete_many_request(server(), term()) -> request_id(). delete_many_request(Server, Keys) -> - gen_server:send_request(Server, {delete, Keys}). - --spec wait_response(request_id(), non_neg_integer() | timeout) -> term(). -wait_response(RequestId, Timeout) -> - case gen_server:wait_response(RequestId, Timeout) of - {reply, {ok, WaitInfo}} -> - wait_for_updated(WaitInfo); - Other -> - Other - end. + async_operation(Server, {delete, Keys}). other_servers(Server) -> gen_server:call(Server, other_servers). @@ -135,13 +122,13 @@ other_nodes(Server) -> other_pids(Server) -> other_servers(Server). --spec pause(server()) -> term(). +-spec pause(server()) -> reference(). pause(Server) -> short_call(Server, pause). --spec unpause(server()) -> term(). -unpause(Server) -> - short_call(Server, unpause). +-spec unpause(server(), reference()) -> term(). +unpause(Server, PauseRef) -> + short_call(Server, {unpause, PauseRef}). %% Waits till all pending operations are applied. -spec sync(server()) -> term(). @@ -167,14 +154,10 @@ init({Tab, Opts}) -> cets_mon_cleaner:start_link(MonTab, MonTab), {ok, #{tab => Tab, mon_tab => MonTab, other_servers => [], opts => Opts, backlog => [], - paused => false, pause_monitor => undefined}}. + pause_monitors => []}}. -spec handle_call(term(), from(), state()) -> {noreply, state()} | {reply, term(), state()}. -handle_call({insert, Rec}, From, State = #{paused := false}) -> - handle_insert(Rec, From, State); -handle_call({delete, Keys}, From, State = #{paused := false}) -> - handle_delete(Keys, From, State); handle_call(other_servers, _From, State = #{other_servers := Servers}) -> {reply, Servers, State}; handle_call(sync, From, State = #{other_servers := Servers}) -> @@ -194,37 +177,30 @@ handle_call(remote_dump, From, State = #{tab := Tab}) -> {noreply, State}; handle_call({send_dump_to_remote_node, NewPids, Dump}, _From, State) -> handle_send_dump_to_remote_node(NewPids, Dump, State); -handle_call(pause, _From = {FromPid, _}, State) -> +handle_call(pause, _From = {FromPid, _}, State = #{pause_monitors := Mons}) -> + %% We monitor who pauses our server Mon = erlang:monitor(process, FromPid), - {reply, ok, State#{paused := true, pause_monitor := Mon}}; -handle_call(unpause, _From, State) -> - handle_unpause(State); + {reply, Mon, State#{pause_monitors := [Mon|Mons]}}; +handle_call({unpause, Ref}, _From, State) -> + handle_unpause(Ref, State); handle_call(get_info, _From, State) -> - handle_get_info(State); -handle_call(Msg, From, State = #{paused := true, backlog := Backlog}) -> - %% Backlog is a list of pending operation, when our server is paused. - %% The list would be applied, once our server is unpaused. - case should_backlog(Msg) of - true -> - {noreply, State#{backlog := [{Msg, From} | Backlog]}}; - false -> - ?LOG_ERROR(#{what => unexpected_call, msg => Msg, from => From}), - {reply, {error, unexpected_call}, State} - end. + handle_get_info(State). -spec handle_cast(term(), state()) -> {noreply, state()}. +handle_cast({op, From, Msg}, State = #{pause_monitors := []}) -> + handle_op(From, Msg, State), + {noreply, State}; +handle_cast({op, From, Msg}, State = #{pause_monitors := [_|_], backlog := Backlog}) -> + %% Backlog is a list of pending operation, when our server is paused. + %% The list would be applied, once our server is unpaused. + {noreply, State#{backlog := [{Msg, From} | Backlog]}}; handle_cast(Msg, State) -> ?LOG_ERROR(#{what => unexpected_cast, msg => Msg}), {noreply, State}. -spec handle_info(term(), state()) -> {noreply, state()}. -handle_info({remote_insert, Mon, Pid, Rec}, State = #{tab := Tab}) -> - ets:insert(Tab, Rec), - reply_updated(Pid, Mon), - {noreply, State}; -handle_info({remote_delete, Mon, Pid, Keys}, State = #{tab := Tab}) -> - ets_delete_keys(Tab, Keys), - reply_updated(Pid, Mon), +handle_info({remote_op, From, Msg}, State) -> + handle_remote_op(From, Msg, State), {noreply, State}; handle_info({'DOWN', Mon, process, Pid, _Reason}, State) -> handle_down(Mon, Pid, State); @@ -246,12 +222,18 @@ handle_send_dump_to_remote_node(NewPids, Dump, Servers2 = add_servers(NewPids, Servers), {reply, ok, State#{other_servers := Servers2}}. -handle_down(Mon, PausedByPid, State = #{pause_monitor := Mon}) -> - ?LOG_ERROR(#{what => pause_owner_crashed, - state => State, paused_by_pid => PausedByPid}), - {reply, ok, State2} = handle_unpause(State), - {noreply, State2}; -handle_down(_Mon, RemotePid, State = #{other_servers := Servers, mon_tab := MonTab}) -> +handle_down(Mon, Pid, State = #{pause_monitors := Mons}) -> + case lists:member(Mon, Mons) of + true -> + ?LOG_ERROR(#{what => pause_owner_crashed, + state => State, paused_by_pid => Pid}), + {reply, ok, State2} = handle_unpause(Mon, State), + {noreply, State2}; + false -> + handle_down2(Mon, Pid, State) + end. + +handle_down2(_Mon, RemotePid, State = #{other_servers := Servers, mon_tab := MonTab}) -> case lists:member(RemotePid, Servers) of true -> Servers2 = lists:delete(RemotePid, Servers), @@ -306,33 +288,39 @@ ets_delete_keys(_Tab, []) -> has_remote_pid(RemotePid, Servers) -> lists:member(RemotePid, Servers). -reply_updated(Pid, Mon) -> +reply_updated({Mon, Pid}) -> %% We really don't wanna block this process erlang:send(Pid, {updated, Mon, self()}, [noconnect, nosuspend]). send_to_remote(RemotePid, Msg) -> erlang:send(RemotePid, Msg, [noconnect, nosuspend]). -handle_insert(Rec, _From = {FromPid, Mon}, - State = #{tab := Tab, mon_tab := MonTab, other_servers := Servers}) -> - ets:insert(Tab, Rec), - %% Insert to other nodes and block till written - WaitInfo = replicate(Mon, Servers, remote_insert, Rec, FromPid, MonTab), - {reply, {ok, WaitInfo}, State}. - -handle_delete(Keys, _From = {FromPid, Mon}, - State = #{tab := Tab, mon_tab := MonTab, other_servers := Servers}) -> - ets_delete_keys(Tab, Keys), - %% Insert to other nodes and block till written - WaitInfo = replicate(Mon, Servers, remote_delete, Keys, FromPid, MonTab), - {reply, {ok, WaitInfo}, State}. - -replicate(Mon, Servers, Cmd, Payload, FromPid, MonTab) -> +%% Handle operation from a remote node +handle_remote_op(From, Msg, State) -> + do_op(Msg, State), + reply_updated(From). + +%% Apply operation for one local table only +do_op(Msg, #{tab := Tab}) -> + do_table_op(Msg, Tab). + +do_table_op({insert, Rec}, Tab) -> + ets:insert(Tab, Rec); +do_table_op({delete, Keys}, Tab) -> + ets_delete_keys(Tab, Keys). + +%% Handle operation locally and replicate it across the cluster +handle_op(From = {Mon, Pid}, Msg, State) when is_pid(Pid) -> + do_op(Msg, State), + WaitInfo = replicate(From, Msg, State), + Pid ! {cets_reply, Mon, WaitInfo}. + +replicate(From, Msg, #{mon_tab := MonTab, other_servers := Servers}) -> %% Reply would be routed directly to FromPid - Msg = {Cmd, Mon, FromPid, Payload}, - replicate2(Servers, Msg), - ets:insert(MonTab, {Mon, FromPid}), - {Mon, Servers, MonTab}. + Msg2 = {remote_op, From, Msg}, + replicate2(Servers, Msg2), + ets:insert(MonTab, From), + {Servers, MonTab}. replicate2([RemotePid | Servers], Msg) -> send_to_remote(RemotePid, Msg), @@ -340,32 +328,9 @@ replicate2([RemotePid | Servers], Msg) -> replicate2([], _Msg) -> ok. -%% Wait for response from the remote nodes that the operation is completed. -%% remote_down is sent by the local server, if the remote server is down. -wait_for_updated({Mon, Servers, MonTab}) -> - try - wait_for_updated2(Mon, Servers) - after - ets:delete(MonTab, Mon) - end. - -wait_for_updated2(_Mon, []) -> - ok; -wait_for_updated2(Mon, Servers) -> - receive - {updated, Mon, Pid} -> - Servers2 = lists:delete(Pid, Servers), - wait_for_updated2(Mon, Servers2); - %% What happens if the main server dies? - {remote_down, Mon, Pid} -> - Servers2 = lists:delete(Pid, Servers), - wait_for_updated2(Mon, Servers2) - end. - apply_backlog([{Msg, From} | Backlog], State) -> - {reply, Reply, State2} = handle_call(Msg, From, State), - gen_server:reply(From, Reply), - apply_backlog(Backlog, State2); + handle_op(From, Msg, State), + apply_backlog(Backlog, State); apply_backlog([], State) -> State. @@ -378,14 +343,23 @@ short_call(RemotePid, Msg) when is_pid(RemotePid) -> short_call(Name, Msg) when is_atom(Name) -> short_call(whereis(Name), Msg). -%% Theoretically we can support mupltiple pauses (but no need for now because -%% we pause in the global locked function) -handle_unpause(State = #{paused := false}) -> +%% We support multiple pauses +%% Only when all pause requests are unpaused we continue +handle_unpause(_Ref, State = #{pause_monitors := []}) -> {reply, {error, already_unpaused}, State}; -handle_unpause(State = #{backlog := Backlog, pause_monitor := Mon}) -> +handle_unpause(Mon, State = #{backlog := Backlog, pause_monitors := Mons}) -> erlang:demonitor(Mon, [flush]), - State2 = State#{paused := false, backlog := [], pause_monitor := undefined}, - {reply, ok, apply_backlog(lists:reverse(Backlog), State2)}. + Mons2 = lists:delete(Mon, Mons), + State2 = State#{pause_monitors := Mons2}, + State3 = + case Mons2 of + [] -> + apply_backlog(lists:reverse(Backlog), State2), + State2#{backlog := []}; + _ -> + State2 + end, + {reply, ok, State3}. handle_get_info(State = #{tab := Tab, other_servers := Servers}) -> Info = #{table => Tab, @@ -406,6 +380,52 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> ok end. -should_backlog({insert, _}) -> true; -should_backlog({delete, _}) -> true; -should_backlog(_) -> false. +async_operation(Server, Msg) -> + Mon = erlang:monitor(process, Server), + gen_server:cast(Server, {op, {Mon, self()}, Msg}), + Mon. + +sync_operation(Server, Msg) -> + Mon = async_operation(Server, Msg), + %% We monitor the local server until the response from all servers is collected. + wait_response(Mon, infinity). + +-spec wait_response(request_id(), non_neg_integer() | infinity) -> term(). +wait_response(Mon, Timeout) -> + receive + {'DOWN', Mon, process, _Pid, Reason} -> + error({cets_down, Reason}); + {cets_reply, Mon, WaitInfo} -> + wait_for_updated(Mon, WaitInfo) + after Timeout -> + erlang:demonitor(Mon, [flush]), + error(timeout) + end. + +%% Wait for response from the remote nodes that the operation is completed. +%% remote_down is sent by the local server, if the remote server is down. +wait_for_updated(Mon, {Servers, MonTab}) -> + try + wait_for_updated2(Mon, Servers) + after + erlang:demonitor(Mon, [flush]), + ets:delete(MonTab, Mon) + end. + +wait_for_updated2(_Mon, []) -> + ok; +wait_for_updated2(Mon, Servers) -> + receive + {updated, Mon, Pid} -> + %% A replication confirmation from the remote server is received + Servers2 = lists:delete(Pid, Servers), + wait_for_updated2(Mon, Servers2); + {remote_down, Mon, Pid} -> + %% This message is sent by our local server when + %% the remote server is down condition is detected + Servers2 = lists:delete(Pid, Servers), + wait_for_updated2(Mon, Servers2); + {'DOWN', Mon, process, _Pid, Reason} -> + %% Local server is down, this is a critical error + error({cets_down, Reason}) + end. diff --git a/src/cets_join.erl b/src/cets_join.erl index 7ba9cb6d..9d6e4374 100644 --- a/src/cets_join.erl +++ b/src/cets_join.erl @@ -52,7 +52,7 @@ join2(_Info, LocalPid, RemotePid) -> LocPids = [LocalPid | LocalOtherPids], RemPids = [RemotePid | RemoteOtherPids], AllPids = LocPids ++ RemPids, - [cets:pause(Pid) || Pid <- AllPids], + Paused = [{Pid, cets:pause(Pid)} || Pid <- AllPids], %% Merges data from two partitions together. %% Each entry in the table is allowed to be updated by the node that owns %% the key only, so merging is easy. @@ -65,7 +65,7 @@ join2(_Info, LocalPid, RemotePid) -> [cets:send_dump_to_remote_node(Pid, RemPids, RemoteDump) || Pid <- LocPids], ok after - [cets:unpause(Pid) || Pid <- AllPids] + lists:foreach(fun({Pid, Ref}) -> cets:unpause(Pid, Ref) end, Paused) end. remote_or_local_dump(Pid) when node(Pid) =:= node() -> diff --git a/test/cets_SUITE.erl b/test/cets_SUITE.erl index a33c5243..be0ef86b 100644 --- a/test/cets_SUITE.erl +++ b/test/cets_SUITE.erl @@ -130,7 +130,7 @@ handle_down_is_called(_Config) -> events_are_applied_in_the_correct_order_after_unpause(_Config) -> T = t4, {ok, Pid} = cets:start(T, #{}), - ok = cets:pause(Pid), + PauseMon = cets:pause(Pid), R1 = cets:insert_request(T, {1}), R2 = cets:delete_request(T, 1), cets:delete_request(T, 2), @@ -141,7 +141,7 @@ events_are_applied_in_the_correct_order_after_unpause(_Config) -> R3 = cets:insert_request(T, [{6}, {7}]), R4 = cets:delete_many_request(T, [5, 4]), [] = lists:sort(cets:dump(T)), - ok = cets:unpause(Pid), + ok = cets:unpause(Pid, PauseMon), [ok = cets:wait_response(R, 5000) || R <- [R1, R2, R3, R4]], [{2}, {3}, {6}, {7}] = lists:sort(cets:dump(T)). From 8a0799ed14b2332b3a4f8e3842ee9930bb3035f0 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 9 May 2022 15:02:52 +0200 Subject: [PATCH 47/64] Add dialyzer task for CI --- .github/workflows/ci.yml | 26 ++++++++ .gitignore | 2 + rebar.config | 9 +++ src/cets.erl | 130 +++++++++++++++++++++++---------------- src/cets_discovery.erl | 12 +++- src/cets_join.erl | 6 +- src/cets_mon_cleaner.erl | 25 ++++---- test/cets_SUITE.erl | 3 +- 8 files changed, 145 insertions(+), 68 deletions(-) create mode 100644 .gitignore create mode 100644 rebar.config diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09cecdb7..7ed05840 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,3 +22,29 @@ jobs: run: rebar3 compile - name: Run tests run: rebar3 ct --sname=ct1 + + dialyzer: + runs-on: ubuntu-latest + + container: + image: erlang:23 + + steps: + - uses: actions/checkout@v3 + - name: Compile + run: rebar3 compile + - name: Run dialyzer + run: rebar3 dialyzer + + xref: + runs-on: ubuntu-latest + + container: + image: erlang:23 + + steps: + - uses: actions/checkout@v3 + - name: Compile + run: rebar3 compile + - name: Run xref + run: rebar3 xref diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..de33d2da --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +_build/ +rebar3 diff --git a/rebar.config b/rebar.config new file mode 100644 index 00000000..430958df --- /dev/null +++ b/rebar.config @@ -0,0 +1,9 @@ +{dialyzer, [ + {warnings, + [no_return, no_unused, no_improper_lists, no_fun_app, no_match, + no_opaque, no_fail_call, no_contracts, no_behaviours, + no_undefined_callbacks, unmatched_returns, error_handling, + race_conditions, underspecs + % overspecs, specdiffs + ]}]}. + diff --git a/src/cets.erl b/src/cets.erl index 8b553bab..49009142 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -17,20 +17,25 @@ -module(cets). -behaviour(gen_server). --export([start/2, stop/1, insert/2, insert_many/2, delete/2, delete_many/2]). --export([dump/1, remote_dump/1, send_dump_to_remote_node/3, table_name/1]). --export([other_nodes/1, other_pids/1]). --export([pause/1, unpause/2, sync/1, ping/1]). --export([info/1]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, +-export([start/2, stop/1, insert/2, insert_many/2, delete/2, delete_many/2, + dump/1, remote_dump/1, send_dump/3, table_name/1, + other_nodes/1, other_pids/1, + pause/1, unpause/2, sync/1, ping/1, info/1, + insert_request/2, insert_many_request/2, + delete_request/2, delete_many_request/2, wait_response/2, + init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --export([insert_request/2, insert_many_request/2, - delete_request/2, delete_many_request/2, wait_response/2]). + +-ignore_xref([start/2, stop/1, insert/2, insert_many/2, delete/2, delete_many/2, + pause/1, unpause/2, sync/1, ping/1, info/1, other_nodes/1, + insert_request/2, insert_many_request/2, + delete_request/2, delete_many_request/2, wait_response/2]). -include_lib("kernel/include/logger.hrl"). --type server() :: atom() | pid(). --type request_id() :: term(). +-type server_ref() :: pid() | atom() | {local, atom()} + | {global, term()} | {via, module(), term()}. +-type request_id() :: reference(). -type backlog_msg() :: {insert, term()} | {delete, term()}. -type from() :: {pid(), reference()}. -type backlog_entry() :: {backlog_msg(), from()}. @@ -42,6 +47,9 @@ backlog := [backlog_entry()], pause_monitors := [reference()]}. +-type short_msg() :: + pause | ping | remote_dump | sync | table_name | {unpause, reference()}. + %% API functions %% Table and server has the same name @@ -61,7 +69,7 @@ stop(Tab) -> dump(Tab) -> ets:tab2list(Tab). --spec remote_dump(server()) -> term(). +-spec remote_dump(server_ref()) -> term(). remote_dump(Server) -> short_call(Server, remote_dump). @@ -70,45 +78,45 @@ table_name(Tab) when is_atom(Tab) -> table_name(Server) -> short_call(Server, table_name). -send_dump_to_remote_node(RemotePid, NewPids, OurDump) -> - Msg = {send_dump_to_remote_node, NewPids, OurDump}, +send_dump(RemotePid, NewPids, OurDump) -> + Msg = {send_dump, NewPids, OurDump}, F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, - Info = #{task => send_dump_to_remote_node, + Info = #{task => send_dump, remote_pid => RemotePid, count => length(OurDump)}, cets_long:run_safely(Info, F). %% Only the node that owns the data could update/remove the data. %% Ideally Key should contain inserter node info (for cleaning). --spec insert(server(), tuple()) -> ok. +-spec insert(server_ref(), tuple()) -> ok. insert(Server, Rec) when is_tuple(Rec) -> sync_operation(Server, {insert, Rec}). --spec insert_many(server(), list(tuple())) -> ok. +-spec insert_many(server_ref(), list(tuple())) -> ok. insert_many(Server, Records) when is_list(Records) -> sync_operation(Server, {insert, Records}). --spec delete(server(), term()) -> ok. +-spec delete(server_ref(), term()) -> ok. delete(Server, Key) -> delete_many(Server, [Key]). %% A separate function for multidelete (because key COULD be a list, so no confusion) --spec delete_many(server(), [term()]) -> ok. +-spec delete_many(server_ref(), [term()]) -> ok. delete_many(Server, Keys) -> sync_operation(Server, {delete, Keys}). --spec insert_request(server(), tuple()) -> request_id(). +-spec insert_request(server_ref(), tuple()) -> request_id(). insert_request(Server, Rec) -> async_operation(Server, {insert, Rec}). --spec insert_many_request(server(), [tuple()]) -> request_id(). +-spec insert_many_request(server_ref(), [tuple()]) -> request_id(). insert_many_request(Server, Records) -> async_operation(Server, {insert, Records}). --spec delete_request(server(), term()) -> request_id(). -delete_request(Tab, Key) -> - delete_many_request(Tab, [Key]). +-spec delete_request(server_ref(), term()) -> request_id(). +delete_request(Server, Key) -> + delete_many_request(Server, [Key]). --spec delete_many_request(server(), term()) -> request_id(). +-spec delete_many_request(server_ref(), [term()]) -> request_id(). delete_many_request(Server, Keys) -> async_operation(Server, {delete, Keys}). @@ -118,28 +126,28 @@ other_servers(Server) -> other_nodes(Server) -> lists:usort(pids_to_nodes(other_pids(Server))). --spec other_pids(server()) -> [pid()]. +-spec other_pids(server_ref()) -> [pid()]. other_pids(Server) -> other_servers(Server). --spec pause(server()) -> reference(). +-spec pause(server_ref()) -> reference(). pause(Server) -> short_call(Server, pause). --spec unpause(server(), reference()) -> term(). +-spec unpause(server_ref(), reference()) -> term(). unpause(Server, PauseRef) -> short_call(Server, {unpause, PauseRef}). %% Waits till all pending operations are applied. --spec sync(server()) -> term(). +-spec sync(server_ref()) -> term(). sync(Server) -> short_call(Server, sync). --spec ping(server()) -> term(). +-spec ping(server_ref()) -> term(). ping(Server) -> short_call(Server, ping). --spec info(server()) -> term(). +-spec info(server_ref()) -> term(). info(Server) -> gen_server:call(Server, get_info). @@ -149,9 +157,9 @@ info(Server) -> init({Tab, Opts}) -> process_flag(message_queue_data, off_heap), MonTab = list_to_atom(atom_to_list(Tab) ++ "_mon"), - ets:new(Tab, [ordered_set, named_table, public]), - ets:new(MonTab, [public, named_table]), - cets_mon_cleaner:start_link(MonTab, MonTab), + _ = ets:new(Tab, [ordered_set, named_table, public]), + _ = ets:new(MonTab, [public, named_table]), + {ok, _} = cets_mon_cleaner:start_link(MonTab, MonTab), {ok, #{tab => Tab, mon_tab => MonTab, other_servers => [], opts => Opts, backlog => [], pause_monitors => []}}. @@ -163,7 +171,7 @@ handle_call(other_servers, _From, State = #{other_servers := Servers}) -> handle_call(sync, From, State = #{other_servers := Servers}) -> %% Do spawn to avoid any possible deadlocks proc_lib:spawn(fun() -> - [ping(Pid) || Pid <- Servers], + lists:foreach(fun ping/1, Servers), gen_server:reply(From, ok) end), {noreply, State}; @@ -175,8 +183,8 @@ handle_call(remote_dump, From, State = #{tab := Tab}) -> %% Do not block the main process (also reduces GC of the main process) proc_lib:spawn_link(fun() -> gen_server:reply(From, {ok, dump(Tab)}) end), {noreply, State}; -handle_call({send_dump_to_remote_node, NewPids, Dump}, _From, State) -> - handle_send_dump_to_remote_node(NewPids, Dump, State); +handle_call({send_dump, NewPids, Dump}, _From, State) -> + handle_send_dump(NewPids, Dump, State); handle_call(pause, _From = {FromPid, _}, State = #{pause_monitors := Mons}) -> %% We monitor who pauses our server Mon = erlang:monitor(process, FromPid), @@ -216,8 +224,7 @@ code_change(_OldVsn, State, _Extra) -> %% Internal logic -handle_send_dump_to_remote_node(NewPids, Dump, - State = #{tab := Tab, other_servers := Servers}) -> +handle_send_dump(NewPids, Dump, State = #{tab := Tab, other_servers := Servers}) -> ets:insert(Tab, Dump), Servers2 = add_servers(NewPids, Servers), {reply, ok, State#{other_servers := Servers2}}. @@ -313,7 +320,8 @@ do_table_op({delete, Keys}, Tab) -> handle_op(From = {Mon, Pid}, Msg, State) when is_pid(Pid) -> do_op(Msg, State), WaitInfo = replicate(From, Msg, State), - Pid ! {cets_reply, Mon, WaitInfo}. + Pid ! {cets_reply, Mon, WaitInfo}, + ok. replicate(From, Msg, #{mon_tab := MonTab, other_servers := Servers}) -> %% Reply would be routed directly to FromPid @@ -331,17 +339,20 @@ replicate2([], _Msg) -> apply_backlog([{Msg, From} | Backlog], State) -> handle_op(From, Msg, State), apply_backlog(Backlog, State); -apply_backlog([], State) -> - State. +apply_backlog([], _State) -> + ok. --spec short_call(server(), term()) -> term(). -short_call(RemotePid, Msg) when is_pid(RemotePid) -> - F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, - Info = #{task => Msg, - remote_pid => RemotePid, remote_node => node(RemotePid)}, - cets_long:run_safely(Info, F); -short_call(Name, Msg) when is_atom(Name) -> - short_call(whereis(Name), Msg). +-spec short_call(server_ref(), short_msg()) -> term(). +short_call(Server, Msg) -> + case where(Server) of + Pid when is_pid(Pid) -> + Info = #{remote_server => Server, remote_pid => Pid, + remote_node => node(Pid)}, + F = fun() -> gen_server:call(Pid, Msg, infinity) end, + cets_long:run_safely(Info, F); + undefined -> + {error, pid_not_found} + end. %% We support multiple pauses %% Only when all pause requests are unpaused we continue @@ -381,9 +392,17 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> end. async_operation(Server, Msg) -> - Mon = erlang:monitor(process, Server), - gen_server:cast(Server, {op, {Mon, self()}, Msg}), - Mon. + case where(Server) of + Pid when is_pid(Pid) -> + Mon = erlang:monitor(process, Pid), + gen_server:cast(Server, {op, {Mon, self()}, Msg}), + Mon; + undefined -> + Mon = make_ref(), + %% Simulate process down + self() ! {'DOWN', Mon, process, undefined, pid_not_found}, + Mon + end. sync_operation(Server, Msg) -> Mon = async_operation(Server, Msg), @@ -429,3 +448,10 @@ wait_for_updated2(Mon, Servers) -> %% Local server is down, this is a critical error error({cets_down, Reason}) end. + +-spec where(server_ref()) -> pid() | undefined. +where(Pid) when is_pid(Pid) -> Pid; +where(Name) when is_atom(Name) -> whereis(Name); +where({global, Name}) -> global:whereis_name(Name); +where({local, Name}) -> whereis(Name); +where({via, Module, Name}) -> Module:whereis_name(Name). diff --git a/src/cets_discovery.erl b/src/cets_discovery.erl index 216556c3..2d24ea06 100644 --- a/src/cets_discovery.erl +++ b/src/cets_discovery.erl @@ -6,6 +6,8 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-ignore_xref([start/1, start_link/1, add_table/2, info/1, behaviour_info/1]). + -include_lib("kernel/include/logger.hrl"). -type backend_state() :: term(). @@ -114,10 +116,15 @@ schedule_check(State) -> State#{timer_ref := TimerRef}. cancel_old_timer(#{timer_ref := OldRef}) when is_reference(OldRef) -> - erlang:cancel_timer(OldRef); + _ = erlang:cancel_timer(OldRef), + flush_all_checks(), + ok; cancel_old_timer(_State) -> ok. +flush_all_checks() -> + receive check -> flush_all_checks() after 0 -> ok end. + do_join(Tab, Node) -> LocalPid = whereis(Tab), %% That would trigger autoconnect for the first time @@ -131,7 +138,8 @@ do_join(Tab, Node) -> report_results(Results, _State = #{results := OldResults}) -> Changed = Results -- OldResults, - [report_result(Result) || Result <- Changed]. + lists:foreach(fun report_result/1, Changed), + ok. report_result(Map) -> ?LOG_INFO(Map). diff --git a/src/cets_join.erl b/src/cets_join.erl index 9d6e4374..f285c380 100644 --- a/src/cets_join.erl +++ b/src/cets_join.erl @@ -61,8 +61,10 @@ join2(_Info, LocalPid, RemotePid) -> cets:sync(RemotePid), {ok, LocalDump} = remote_or_local_dump(LocalPid), {ok, RemoteDump} = remote_or_local_dump(RemotePid), - [cets:send_dump_to_remote_node(Pid, LocPids, LocalDump) || Pid <- RemPids], - [cets:send_dump_to_remote_node(Pid, RemPids, RemoteDump) || Pid <- LocPids], + RemF = fun(Pid) -> cets:send_dump(Pid, LocPids, LocalDump) end, + LocF = fun(Pid) -> cets:send_dump(Pid, RemPids, RemoteDump) end, + lists:foreach(RemF, RemPids), + lists:foreach(LocF, LocPids), ok after lists:foreach(fun({Pid, Ref}) -> cets:unpause(Pid, Ref) end, Paused) diff --git a/src/cets_mon_cleaner.erl b/src/cets_mon_cleaner.erl index b9f03017..860a9971 100644 --- a/src/cets_mon_cleaner.erl +++ b/src/cets_mon_cleaner.erl @@ -12,7 +12,7 @@ -include_lib("kernel/include/logger.hrl"). --type timer_ref() :: reference() | undefined. +-type timer_ref() :: reference(). -type state() :: #{ mon_tab := atom(), interval := non_neg_integer(), @@ -24,8 +24,10 @@ start_link(Name, MonTab) -> -spec init(atom()) -> {ok, state()}. init(MonTab) -> - State = #{mon_tab => MonTab, interval => 30000, timer_ref => undefined}, - {ok, schedule_check(State)}. + Interval = 30000, + State = #{mon_tab => MonTab, interval => Interval, + timer_ref => start_timer(Interval)}, + {ok, State}. handle_call(Msg, From, State) -> ?LOG_ERROR(#{what => unexpected_call, msg => Msg, from => From}), @@ -48,15 +50,16 @@ code_change(_OldVsn, State, _Extra) -> {ok, State}. -spec schedule_check(state()) -> state(). -schedule_check(State = #{interval := Interval}) -> - cancel_old_timer(State), - TimerRef = erlang:send_after(Interval, self(), check), - State#{timer_ref := TimerRef}. +schedule_check(State = #{interval := Interval, timer_ref := OldRef}) -> + _ = erlang:cancel_timer(OldRef), + flush_all_checks(), + State#{timer_ref := start_timer(Interval)}. -cancel_old_timer(#{timer_ref := OldRef}) when is_reference(OldRef) -> - erlang:cancel_timer(OldRef); -cancel_old_timer(_State) -> - ok. +flush_all_checks() -> + receive check -> flush_all_checks() after 0 -> ok end. + +start_timer(Interval) -> + erlang:send_after(Interval, self(), check). -spec handle_check(state()) -> state(). handle_check(State = #{mon_tab := MonTab}) -> diff --git a/test/cets_SUITE.erl b/test/cets_SUITE.erl index be0ef86b..ed1899be 100644 --- a/test/cets_SUITE.erl +++ b/test/cets_SUITE.erl @@ -54,7 +54,8 @@ test_multinode(Config) -> X = dump(Node1, Tab), X = dump(Node2, Tab), X = dump(Node3, Tab), - X = dump(Node4, Tab) + X = dump(Node4, Tab), + ok end, Same([{a}, {b}, {c}, {d}, {e}, {f}]), delete(Node1, Tab, e), From e7a25c1f410194b1630258afe48e95c9a2f306e7 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 9 May 2022 18:56:12 +0200 Subject: [PATCH 48/64] Separate operations for insert and insert_many Separate operations for delete and delete_many Add more types --- src/cets.erl | 69 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/src/cets.erl b/src/cets.erl index 49009142..54601dc9 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -36,20 +36,31 @@ -type server_ref() :: pid() | atom() | {local, atom()} | {global, term()} | {via, module(), term()}. -type request_id() :: reference(). --type backlog_msg() :: {insert, term()} | {delete, term()}. +-type op() :: {insert, tuple()} | {delete, term()} + | {insert_many, [tuple()]} | {delete_many, [term()]}. -type from() :: {pid(), reference()}. --type backlog_entry() :: {backlog_msg(), from()}. +-type backlog_entry() :: {op(), from()}. +-type table_name() :: atom(). +-type pause_monitor() :: reference(). -type state() :: #{ - tab := atom(), + tab := table_name(), mon_tab := atom(), other_servers := [pid()], - opts := map(), + opts := start_opts(), backlog := [backlog_entry()], - pause_monitors := [reference()]}. + pause_monitors := [pause_monitor()]}. -type short_msg() :: pause | ping | remote_dump | sync | table_name | {unpause, reference()}. +-type info() :: #{table := table_name(), + nodes := [node()], + size := non_neg_integer(), + memory := non_neg_integer()}. + +-type handle_down_fun() :: fun((#{remote_pid := pid(), table := table_name()}) -> ok). +-type start_opts() :: #{handle_down := handle_down_fun()}. + %% API functions %% Table and server has the same name @@ -60,24 +71,29 @@ %% to make a new async process if you need to update). %% i.e. any functions that replicate changes are not allowed (i.e. insert/2, %% remove/2). +-spec start(table_name(), start_opts()) -> {ok, pid()}. start(Tab, Opts) when is_atom(Tab) -> gen_server:start({local, Tab}, ?MODULE, {Tab, Opts}, []). -stop(Tab) -> - gen_server:stop(Tab). +-spec stop(server_ref()) -> ok. +stop(Server) -> + gen_server:stop(Server). +-spec dump(server_ref()) -> Records :: [tuple()]. dump(Tab) -> ets:tab2list(Tab). --spec remote_dump(server_ref()) -> term(). +-spec remote_dump(server_ref()) -> {ok, Records :: [tuple()]}. remote_dump(Server) -> short_call(Server, remote_dump). +-spec table_name(server_ref()) -> table_name(). table_name(Tab) when is_atom(Tab) -> Tab; table_name(Server) -> short_call(Server, table_name). +-spec send_dump(pid(), [pid()], [tuple()]) -> ok. send_dump(RemotePid, NewPids, OurDump) -> Msg = {send_dump, NewPids, OurDump}, F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, @@ -93,16 +109,16 @@ insert(Server, Rec) when is_tuple(Rec) -> -spec insert_many(server_ref(), list(tuple())) -> ok. insert_many(Server, Records) when is_list(Records) -> - sync_operation(Server, {insert, Records}). + sync_operation(Server, {insert_many, Records}). -spec delete(server_ref(), term()) -> ok. delete(Server, Key) -> - delete_many(Server, [Key]). + sync_operation(Server, {delete, Key}). %% A separate function for multidelete (because key COULD be a list, so no confusion) -spec delete_many(server_ref(), [term()]) -> ok. delete_many(Server, Keys) -> - sync_operation(Server, {delete, Keys}). + sync_operation(Server, {delete_many, Keys}). -spec insert_request(server_ref(), tuple()) -> request_id(). insert_request(Server, Rec) -> @@ -110,19 +126,21 @@ insert_request(Server, Rec) -> -spec insert_many_request(server_ref(), [tuple()]) -> request_id(). insert_many_request(Server, Records) -> - async_operation(Server, {insert, Records}). + async_operation(Server, {insert_many, Records}). -spec delete_request(server_ref(), term()) -> request_id(). delete_request(Server, Key) -> - delete_many_request(Server, [Key]). + async_operation(Server, {delete, Key}). -spec delete_many_request(server_ref(), [term()]) -> request_id(). delete_many_request(Server, Keys) -> - async_operation(Server, {delete, Keys}). + async_operation(Server, {delete_many, Keys}). +-spec other_servers(server_ref()) -> [server_ref()]. other_servers(Server) -> gen_server:call(Server, other_servers). +-spec other_nodes(server_ref()) -> [node()]. other_nodes(Server) -> lists:usort(pids_to_nodes(other_pids(Server))). @@ -130,30 +148,30 @@ other_nodes(Server) -> other_pids(Server) -> other_servers(Server). --spec pause(server_ref()) -> reference(). +-spec pause(server_ref()) -> pause_monitor(). pause(Server) -> short_call(Server, pause). --spec unpause(server_ref(), reference()) -> term(). +-spec unpause(server_ref(), pause_monitor()) -> ok. unpause(Server, PauseRef) -> short_call(Server, {unpause, PauseRef}). %% Waits till all pending operations are applied. --spec sync(server_ref()) -> term(). +-spec sync(server_ref()) -> ok. sync(Server) -> short_call(Server, sync). --spec ping(server_ref()) -> term(). +-spec ping(server_ref()) -> pong. ping(Server) -> short_call(Server, ping). --spec info(server_ref()) -> term(). +-spec info(server_ref()) -> info(). info(Server) -> gen_server:call(Server, get_info). %% gen_server callbacks --spec init(term()) -> {ok, state()}. +-spec init({table_name(), start_opts()}) -> {ok, state()}. init({Tab, Opts}) -> process_flag(message_queue_data, off_heap), MonTab = list_to_atom(atom_to_list(Tab) ++ "_mon"), @@ -176,7 +194,7 @@ handle_call(sync, From, State = #{other_servers := Servers}) -> end), {noreply, State}; handle_call(ping, _From, State) -> - {reply, ping, State}; + {reply, pong, State}; handle_call(table_name, _From, State = #{tab := Tab}) -> {reply, {ok, Tab}, State}; handle_call(remote_dump, From, State = #{tab := Tab}) -> @@ -313,7 +331,11 @@ do_op(Msg, #{tab := Tab}) -> do_table_op({insert, Rec}, Tab) -> ets:insert(Tab, Rec); -do_table_op({delete, Keys}, Tab) -> +do_table_op({delete, Key}, Tab) -> + ets:delete(Tab, Key); +do_table_op({insert_many, Recs}, Tab) -> + ets:insert(Tab, Recs); +do_table_op({delete_many, Keys}, Tab) -> ets_delete_keys(Tab, Keys). %% Handle operation locally and replicate it across the cluster @@ -372,6 +394,7 @@ handle_unpause(Mon, State = #{backlog := Backlog, pause_monitors := Mons}) -> end, {reply, ok, State3}. +-spec handle_get_info(state()) -> {reply, info(), state()}. handle_get_info(State = #{tab := Tab, other_servers := Servers}) -> Info = #{table => Tab, nodes => lists:usort(pids_to_nodes([self() | Servers])), @@ -391,6 +414,7 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> ok end. +-spec async_operation(server_ref(), op()) -> request_id(). async_operation(Server, Msg) -> case where(Server) of Pid when is_pid(Pid) -> @@ -404,6 +428,7 @@ async_operation(Server, Msg) -> Mon end. +-spec sync_operation(server_ref(), op()) -> ok. sync_operation(Server, Msg) -> Mon = async_operation(Server, Msg), %% We monitor the local server until the response from all servers is collected. From e20324398e7876030e3370b1259b56c278df5be7 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 9 May 2022 19:21:45 +0200 Subject: [PATCH 49/64] Add unpause_twice and pause_multiple_times testcases --- src/cets.erl | 27 ++++++++++++++++++--------- test/cets_SUITE.erl | 26 ++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/cets.erl b/src/cets.erl index 54601dc9..ad909b7f 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -152,7 +152,7 @@ other_pids(Server) -> pause(Server) -> short_call(Server, pause). --spec unpause(server_ref(), pause_monitor()) -> ok. +-spec unpause(server_ref(), pause_monitor()) -> ok | {error, unknown_pause_monitor}. unpause(Server, PauseRef) -> short_call(Server, {unpause, PauseRef}). @@ -358,10 +358,14 @@ replicate2([RemotePid | Servers], Msg) -> replicate2([], _Msg) -> ok. -apply_backlog([{Msg, From} | Backlog], State) -> +apply_backlog(State = #{backlog := Backlog}) -> + apply_backlog_ops(lists:reverse(Backlog), State), + State#{backlog := []}. + +apply_backlog_ops([{Msg, From} | Backlog], State) -> handle_op(From, Msg, State), - apply_backlog(Backlog, State); -apply_backlog([], _State) -> + apply_backlog_ops(Backlog, State); +apply_backlog_ops([], _State) -> ok. -spec short_call(server_ref(), short_msg()) -> term(). @@ -378,17 +382,22 @@ short_call(Server, Msg) -> %% We support multiple pauses %% Only when all pause requests are unpaused we continue -handle_unpause(_Ref, State = #{pause_monitors := []}) -> - {reply, {error, already_unpaused}, State}; -handle_unpause(Mon, State = #{backlog := Backlog, pause_monitors := Mons}) -> +handle_unpause(Mon, State = #{pause_monitors := Mons}) -> + case lists:member(Mon, Mons) of + true -> + handle_unpause2(Mon, Mons, State); + false -> + {reply, {error, unknown_pause_monitor}, State} + end. + +handle_unpause2(Mon, Mons, State) -> erlang:demonitor(Mon, [flush]), Mons2 = lists:delete(Mon, Mons), State2 = State#{pause_monitors := Mons2}, State3 = case Mons2 of [] -> - apply_backlog(lists:reverse(Backlog), State2), - State2#{backlog := []}; + apply_backlog(State2); _ -> State2 end, diff --git a/test/cets_SUITE.erl b/test/cets_SUITE.erl index ed1899be..d8d4b82d 100644 --- a/test/cets_SUITE.erl +++ b/test/cets_SUITE.erl @@ -7,6 +7,8 @@ all() -> [test_multinode, node_list_is_correct, test_multinode_auto_discovery, test_locally, handle_down_is_called, events_are_applied_in_the_correct_order_after_unpause, + pause_multiple_times, + unpause_twice, write_returns_if_remote_server_crashes, mon_cleaner_works, sync_using_name_works, insert_many_request]. @@ -146,6 +148,30 @@ events_are_applied_in_the_correct_order_after_unpause(_Config) -> [ok = cets:wait_response(R, 5000) || R <- [R1, R2, R3, R4]], [{2}, {3}, {6}, {7}] = lists:sort(cets:dump(T)). +pause_multiple_times(_Config) -> + T = t5, + {ok, Pid} = cets:start(T, #{}), + PauseMon1 = cets:pause(Pid), + PauseMon2 = cets:pause(Pid), + Ref1 = cets:insert_request(Pid, {1}), + Ref2 = cets:insert_request(Pid, {2}), + [] = cets:dump(T), %% No records yet, even after pong + ok = cets:unpause(Pid, PauseMon1), + pong = cets:ping(Pid), + [] = cets:dump(T), %% No records yet, even after pong + ok = cets:unpause(Pid, PauseMon2), + pong = cets:ping(Pid), + cets:wait_response(Ref1, 5000), + cets:wait_response(Ref2, 5000), + [{1}, {2}] = lists:sort(cets:dump(T)). + +unpause_twice(_Config) -> + T = t6, + {ok, Pid} = cets:start(T, #{}), + PauseMon = cets:pause(Pid), + ok = cets:unpause(Pid, PauseMon), + {error, unknown_pause_monitor} = cets:unpause(Pid, PauseMon). + write_returns_if_remote_server_crashes(_Config) -> {ok, Pid1} = cets:start(c1, #{}), {ok, Pid2} = cets:start(c2, #{}), From 36fc17f13526c69f2129d1da040e984e92531a58 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 9 May 2022 19:54:51 +0200 Subject: [PATCH 50/64] Use long_call function --- src/cets.erl | 58 ++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/cets.erl b/src/cets.erl index ad909b7f..74520a7c 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -50,8 +50,8 @@ backlog := [backlog_entry()], pause_monitors := [pause_monitor()]}. --type short_msg() :: - pause | ping | remote_dump | sync | table_name | {unpause, reference()}. +-type long_msg() :: pause | ping | remote_dump | sync | table_name | get_info + | other_servers | {unpause, reference()}. -type info() :: #{table := table_name(), nodes := [node()], @@ -85,21 +85,18 @@ dump(Tab) -> -spec remote_dump(server_ref()) -> {ok, Records :: [tuple()]}. remote_dump(Server) -> - short_call(Server, remote_dump). + long_call(Server, remote_dump). -spec table_name(server_ref()) -> table_name(). table_name(Tab) when is_atom(Tab) -> Tab; table_name(Server) -> - short_call(Server, table_name). + long_call(Server, table_name). --spec send_dump(pid(), [pid()], [tuple()]) -> ok. -send_dump(RemotePid, NewPids, OurDump) -> - Msg = {send_dump, NewPids, OurDump}, - F = fun() -> gen_server:call(RemotePid, Msg, infinity) end, - Info = #{task => send_dump, - remote_pid => RemotePid, count => length(OurDump)}, - cets_long:run_safely(Info, F). +-spec send_dump(server_ref(), [pid()], [tuple()]) -> ok. +send_dump(Server, NewPids, OurDump) -> + Info = #{msg => send_dump, count => length(OurDump)}, + long_call(Server, {send_dump, NewPids, OurDump}, Info). %% Only the node that owns the data could update/remove the data. %% Ideally Key should contain inserter node info (for cleaning). @@ -138,7 +135,7 @@ delete_many_request(Server, Keys) -> -spec other_servers(server_ref()) -> [server_ref()]. other_servers(Server) -> - gen_server:call(Server, other_servers). + long_call(Server, other_servers). -spec other_nodes(server_ref()) -> [node()]. other_nodes(Server) -> @@ -150,24 +147,24 @@ other_pids(Server) -> -spec pause(server_ref()) -> pause_monitor(). pause(Server) -> - short_call(Server, pause). + long_call(Server, pause). -spec unpause(server_ref(), pause_monitor()) -> ok | {error, unknown_pause_monitor}. unpause(Server, PauseRef) -> - short_call(Server, {unpause, PauseRef}). + long_call(Server, {unpause, PauseRef}). %% Waits till all pending operations are applied. -spec sync(server_ref()) -> ok. sync(Server) -> - short_call(Server, sync). + long_call(Server, sync). -spec ping(server_ref()) -> pong. ping(Server) -> - short_call(Server, ping). + long_call(Server, ping). -spec info(server_ref()) -> info(). info(Server) -> - gen_server:call(Server, get_info). + long_call(Server, get_info). %% gen_server callbacks @@ -368,18 +365,6 @@ apply_backlog_ops([{Msg, From} | Backlog], State) -> apply_backlog_ops([], _State) -> ok. --spec short_call(server_ref(), short_msg()) -> term(). -short_call(Server, Msg) -> - case where(Server) of - Pid when is_pid(Pid) -> - Info = #{remote_server => Server, remote_pid => Pid, - remote_node => node(Pid)}, - F = fun() -> gen_server:call(Pid, Msg, infinity) end, - cets_long:run_safely(Info, F); - undefined -> - {error, pid_not_found} - end. - %% We support multiple pauses %% Only when all pause requests are unpaused we continue handle_unpause(Mon, State = #{pause_monitors := Mons}) -> @@ -423,6 +408,21 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> ok end. +-spec long_call(server_ref(), long_msg()) -> term(). +long_call(Server, Msg) -> + long_call(Server, Msg, #{msg => Msg}). + +long_call(Server, Msg, Info) -> + case where(Server) of + Pid when is_pid(Pid) -> + Info2 = Info#{remote_server => Server, remote_pid => Pid, + remote_node => node(Pid)}, + F = fun() -> gen_server:call(Pid, Msg, infinity) end, + cets_long:run_safely(Info2, F); + undefined -> + {error, pid_not_found} + end. + -spec async_operation(server_ref(), op()) -> request_id(). async_operation(Server, Msg) -> case where(Server) of From 90e1e65296627a4087b7795832f8e54bd6daecd0 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 19 Oct 2022 12:42:12 +0200 Subject: [PATCH 51/64] Move call specific code into cets_call To split the business logic from the helpers logic --- .gitignore | 2 + rebar.config | 2 +- src/cets.erl | 122 +++++++++------------------------------------- src/cets_call.erl | 109 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 100 deletions(-) create mode 100644 src/cets_call.erl diff --git a/.gitignore b/.gitignore index de33d2da..9ccc8e03 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ _build/ rebar3 +# vim temporary files +*.sw* diff --git a/rebar.config b/rebar.config index 430958df..c2f77ada 100644 --- a/rebar.config +++ b/rebar.config @@ -3,7 +3,7 @@ [no_return, no_unused, no_improper_lists, no_fun_app, no_match, no_opaque, no_fail_call, no_contracts, no_behaviours, no_undefined_callbacks, unmatched_returns, error_handling, - race_conditions, underspecs + underspecs % overspecs, specdiffs ]}]}. diff --git a/src/cets.erl b/src/cets.erl index 74520a7c..46c4ec62 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -61,6 +61,8 @@ -type handle_down_fun() :: fun((#{remote_pid := pid(), table := table_name()}) -> ok). -type start_opts() :: #{handle_down := handle_down_fun()}. +-export_type([request_id/0, op/0, server_ref/0, long_msg/0]). + %% API functions %% Table and server has the same name @@ -85,57 +87,61 @@ dump(Tab) -> -spec remote_dump(server_ref()) -> {ok, Records :: [tuple()]}. remote_dump(Server) -> - long_call(Server, remote_dump). + cets_call:long_call(Server, remote_dump). -spec table_name(server_ref()) -> table_name(). table_name(Tab) when is_atom(Tab) -> Tab; table_name(Server) -> - long_call(Server, table_name). + cets_call:long_call(Server, table_name). -spec send_dump(server_ref(), [pid()], [tuple()]) -> ok. send_dump(Server, NewPids, OurDump) -> Info = #{msg => send_dump, count => length(OurDump)}, - long_call(Server, {send_dump, NewPids, OurDump}, Info). + cets_call:long_call(Server, {send_dump, NewPids, OurDump}, Info). %% Only the node that owns the data could update/remove the data. %% Ideally Key should contain inserter node info (for cleaning). -spec insert(server_ref(), tuple()) -> ok. insert(Server, Rec) when is_tuple(Rec) -> - sync_operation(Server, {insert, Rec}). + cets_call:sync_operation(Server, {insert, Rec}). -spec insert_many(server_ref(), list(tuple())) -> ok. insert_many(Server, Records) when is_list(Records) -> - sync_operation(Server, {insert_many, Records}). + cets_call:sync_operation(Server, {insert_many, Records}). -spec delete(server_ref(), term()) -> ok. delete(Server, Key) -> - sync_operation(Server, {delete, Key}). + cets_call:sync_operation(Server, {delete, Key}). %% A separate function for multidelete (because key COULD be a list, so no confusion) -spec delete_many(server_ref(), [term()]) -> ok. delete_many(Server, Keys) -> - sync_operation(Server, {delete_many, Keys}). + cets_call:sync_operation(Server, {delete_many, Keys}). -spec insert_request(server_ref(), tuple()) -> request_id(). insert_request(Server, Rec) -> - async_operation(Server, {insert, Rec}). + cets_call:async_operation(Server, {insert, Rec}). -spec insert_many_request(server_ref(), [tuple()]) -> request_id(). insert_many_request(Server, Records) -> - async_operation(Server, {insert_many, Records}). + cets_call:async_operation(Server, {insert_many, Records}). -spec delete_request(server_ref(), term()) -> request_id(). delete_request(Server, Key) -> - async_operation(Server, {delete, Key}). + cets_call:async_operation(Server, {delete, Key}). -spec delete_many_request(server_ref(), [term()]) -> request_id(). delete_many_request(Server, Keys) -> - async_operation(Server, {delete_many, Keys}). + cets_call:async_operation(Server, {delete_many, Keys}). + +-spec wait_response(request_id(), non_neg_integer() | infinity) -> ok. +wait_response(Mon, Timeout) -> + cets_call:wait_response(Mon, Timeout). -spec other_servers(server_ref()) -> [server_ref()]. other_servers(Server) -> - long_call(Server, other_servers). + cets_call:long_call(Server, other_servers). -spec other_nodes(server_ref()) -> [node()]. other_nodes(Server) -> @@ -147,24 +153,24 @@ other_pids(Server) -> -spec pause(server_ref()) -> pause_monitor(). pause(Server) -> - long_call(Server, pause). + cets_call:long_call(Server, pause). -spec unpause(server_ref(), pause_monitor()) -> ok | {error, unknown_pause_monitor}. unpause(Server, PauseRef) -> - long_call(Server, {unpause, PauseRef}). + cets_call:long_call(Server, {unpause, PauseRef}). %% Waits till all pending operations are applied. -spec sync(server_ref()) -> ok. sync(Server) -> - long_call(Server, sync). + cets_call:long_call(Server, sync). -spec ping(server_ref()) -> pong. ping(Server) -> - long_call(Server, ping). + cets_call:long_call(Server, ping). -spec info(server_ref()) -> info(). info(Server) -> - long_call(Server, get_info). + cets_call:long_call(Server, get_info). %% gen_server callbacks @@ -407,85 +413,3 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> _ -> ok end. - --spec long_call(server_ref(), long_msg()) -> term(). -long_call(Server, Msg) -> - long_call(Server, Msg, #{msg => Msg}). - -long_call(Server, Msg, Info) -> - case where(Server) of - Pid when is_pid(Pid) -> - Info2 = Info#{remote_server => Server, remote_pid => Pid, - remote_node => node(Pid)}, - F = fun() -> gen_server:call(Pid, Msg, infinity) end, - cets_long:run_safely(Info2, F); - undefined -> - {error, pid_not_found} - end. - --spec async_operation(server_ref(), op()) -> request_id(). -async_operation(Server, Msg) -> - case where(Server) of - Pid when is_pid(Pid) -> - Mon = erlang:monitor(process, Pid), - gen_server:cast(Server, {op, {Mon, self()}, Msg}), - Mon; - undefined -> - Mon = make_ref(), - %% Simulate process down - self() ! {'DOWN', Mon, process, undefined, pid_not_found}, - Mon - end. - --spec sync_operation(server_ref(), op()) -> ok. -sync_operation(Server, Msg) -> - Mon = async_operation(Server, Msg), - %% We monitor the local server until the response from all servers is collected. - wait_response(Mon, infinity). - --spec wait_response(request_id(), non_neg_integer() | infinity) -> term(). -wait_response(Mon, Timeout) -> - receive - {'DOWN', Mon, process, _Pid, Reason} -> - error({cets_down, Reason}); - {cets_reply, Mon, WaitInfo} -> - wait_for_updated(Mon, WaitInfo) - after Timeout -> - erlang:demonitor(Mon, [flush]), - error(timeout) - end. - -%% Wait for response from the remote nodes that the operation is completed. -%% remote_down is sent by the local server, if the remote server is down. -wait_for_updated(Mon, {Servers, MonTab}) -> - try - wait_for_updated2(Mon, Servers) - after - erlang:demonitor(Mon, [flush]), - ets:delete(MonTab, Mon) - end. - -wait_for_updated2(_Mon, []) -> - ok; -wait_for_updated2(Mon, Servers) -> - receive - {updated, Mon, Pid} -> - %% A replication confirmation from the remote server is received - Servers2 = lists:delete(Pid, Servers), - wait_for_updated2(Mon, Servers2); - {remote_down, Mon, Pid} -> - %% This message is sent by our local server when - %% the remote server is down condition is detected - Servers2 = lists:delete(Pid, Servers), - wait_for_updated2(Mon, Servers2); - {'DOWN', Mon, process, _Pid, Reason} -> - %% Local server is down, this is a critical error - error({cets_down, Reason}) - end. - --spec where(server_ref()) -> pid() | undefined. -where(Pid) when is_pid(Pid) -> Pid; -where(Name) when is_atom(Name) -> whereis(Name); -where({global, Name}) -> global:whereis_name(Name); -where({local, Name}) -> whereis(Name); -where({via, Module, Name}) -> Module:whereis_name(Name). diff --git a/src/cets_call.erl b/src/cets_call.erl new file mode 100644 index 00000000..11127c38 --- /dev/null +++ b/src/cets_call.erl @@ -0,0 +1,109 @@ +%% @doc Module for extending gen_server calls. +%% Also, it contains code for sync and async multinode operations. +%% Operations are messages which could be buffered when a server is paused. +%% Operations are also broadcasted to the whole cluster. +-module(cets_call). + +-export([long_call/2, long_call/3]). +-export([async_operation/2]). +-export([sync_operation/2]). +-export([wait_response/2]). + +-type request_id() :: cets:request_id(). +-type op() :: cets:op(). +-type server_ref() :: cets:server_ref(). +-type long_msg() :: cets:long_msg(). + +%% Do gen_server:call with better error reporting. +%% It would print a warning if the call takes too long. +-spec long_call(server_ref(), long_msg()) -> term(). +long_call(Server, Msg) -> + long_call(Server, Msg, #{msg => Msg}). + +-spec long_call(server_ref(), long_msg(), map()) -> term(). +long_call(Server, Msg, Info) -> + case where(Server) of + Pid when is_pid(Pid) -> + Info2 = Info#{remote_server => Server, remote_pid => Pid, + remote_node => node(Pid)}, + F = fun() -> gen_server:call(Pid, Msg, infinity) end, + cets_long:run_safely(Info2, F); + undefined -> + {error, pid_not_found} + end. + +%% Contacts the local server to broadcast multinode operation. +%% Returns immediately. +%% You can wait for response from all nodes by calling wait_response/2. +%% You would have to call wait_response/2 to process incoming messages and to remove the monitor +%% (or the caller process can just exit to clean this up). +%% +%% (could not be implemented by an async gen_server:call, because we want +%% to keep monitoring the local gen_server till all responses are received). +-spec async_operation(server_ref(), op()) -> request_id(). +async_operation(Server, Msg) -> + case where(Server) of + Pid when is_pid(Pid) -> + Mon = erlang:monitor(process, Pid), + gen_server:cast(Server, {op, {Mon, self()}, Msg}), + Mon; + undefined -> + Mon = make_ref(), + %% Simulate process down + self() ! {'DOWN', Mon, process, undefined, pid_not_found}, + Mon + end. + +-spec sync_operation(server_ref(), op()) -> ok. +sync_operation(Server, Msg) -> + Mon = async_operation(Server, Msg), + %% We monitor the local server until the response from all servers is collected. + wait_response(Mon, infinity). + +%% This function must be called to receive the result of the multinode operation. +-spec wait_response(request_id(), non_neg_integer() | infinity) -> ok. +wait_response(Mon, Timeout) -> + receive + {'DOWN', Mon, process, _Pid, Reason} -> + error({cets_down, Reason}); + {cets_reply, Mon, WaitInfo} -> + wait_for_updated(Mon, WaitInfo) + after Timeout -> + erlang:demonitor(Mon, [flush]), + error(timeout) + end. + +%% Wait for response from the remote nodes that the operation is completed. +%% remote_down is sent by the local server, if the remote server is down. +wait_for_updated(Mon, {Servers, MonTab}) -> + try + wait_for_updated2(Mon, Servers) + after + erlang:demonitor(Mon, [flush]), + ets:delete(MonTab, Mon) + end. + +wait_for_updated2(_Mon, []) -> + ok; +wait_for_updated2(Mon, Servers) -> + receive + {updated, Mon, Pid} -> + %% A replication confirmation from the remote server is received + Servers2 = lists:delete(Pid, Servers), + wait_for_updated2(Mon, Servers2); + {remote_down, Mon, Pid} -> + %% This message is sent by our local server when + %% the remote server is down condition is detected + Servers2 = lists:delete(Pid, Servers), + wait_for_updated2(Mon, Servers2); + {'DOWN', Mon, process, _Pid, Reason} -> + %% Local server is down, this is a critical error + error({cets_down, Reason}) + end. + +-spec where(server_ref()) -> pid() | undefined. +where(Pid) when is_pid(Pid) -> Pid; +where(Name) when is_atom(Name) -> whereis(Name); +where({global, Name}) -> global:whereis_name(Name); +where({local, Name}) -> whereis(Name); +where({via, Module, Name}) -> Module:whereis_name(Name). From 3cb135f95cc94eacc5d1c14c1c1e648097d8799d Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 19 Oct 2022 18:48:58 +0200 Subject: [PATCH 52/64] Update file docs --- src/cets.erl | 8 ++++++-- src/cets_discovery.erl | 1 + src/cets_discovery_file.erl | 4 +++- src/cets_join.erl | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/cets.erl b/src/cets.erl index 46c4ec62..980e74b5 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -101,7 +101,7 @@ send_dump(Server, NewPids, OurDump) -> cets_call:long_call(Server, {send_dump, NewPids, OurDump}, Info). %% Only the node that owns the data could update/remove the data. -%% Ideally Key should contain inserter node info (for cleaning). +%% Ideally, Key should contain inserter node info so cleaning and merging is simplified. -spec insert(server_ref(), tuple()) -> ok. insert(Server, Rec) when is_tuple(Rec) -> cets_call:sync_operation(Server, {insert, Rec}). @@ -110,6 +110,8 @@ insert(Server, Rec) when is_tuple(Rec) -> insert_many(Server, Records) when is_list(Records) -> cets_call:sync_operation(Server, {insert_many, Records}). +%% Removes an object with the key from all nodes in the cluster. +%% Ideally, nodes should only remove data that they've inserted, not data from other node. -spec delete(server_ref(), term()) -> ok. delete(Server, Key) -> cets_call:sync_operation(Server, {delete, Key}). @@ -139,14 +141,17 @@ delete_many_request(Server, Keys) -> wait_response(Mon, Timeout) -> cets_call:wait_response(Mon, Timeout). +%% Get a list of other CETS processes that are handling this table. -spec other_servers(server_ref()) -> [server_ref()]. other_servers(Server) -> cets_call:long_call(Server, other_servers). +%% Get a list of other nodes in the cluster that are connected together. -spec other_nodes(server_ref()) -> [node()]. other_nodes(Server) -> lists:usort(pids_to_nodes(other_pids(Server))). +%% Get a list of other CETS processes that are handling this table. -spec other_pids(server_ref()) -> [pid()]. other_pids(Server) -> other_servers(Server). @@ -266,7 +271,6 @@ handle_down2(_Mon, RemotePid, State = #{other_servers := Servers, mon_tab := Mon true -> Servers2 = lists:delete(RemotePid, Servers), notify_remote_down(RemotePid, MonTab), - %% Down from a proxy call_user_handle_down(RemotePid, State), {noreply, State#{other_servers := Servers2}}; false -> diff --git a/src/cets_discovery.erl b/src/cets_discovery.erl index 2d24ea06..45a586eb 100644 --- a/src/cets_discovery.erl +++ b/src/cets_discovery.erl @@ -1,3 +1,4 @@ +%% @doc Node discovery logic %% Joins table together when a new node appears -module(cets_discovery). -behaviour(gen_server). diff --git a/src/cets_discovery_file.erl b/src/cets_discovery_file.erl index cdad52f1..198623c4 100644 --- a/src/cets_discovery_file.erl +++ b/src/cets_discovery_file.erl @@ -1,4 +1,6 @@ -%% AWS auto-discovery is kinda bad. +%% @doc File backend for cets_discovery. +%% +%% Barebone AWS EC2 auto-discovery is limited: %% - UDP broadcasts do not work %% - AWS CLI needs access %% - DNS does not allow to list subdomains diff --git a/src/cets_join.erl b/src/cets_join.erl index f285c380..3e400a84 100644 --- a/src/cets_join.erl +++ b/src/cets_join.erl @@ -1,3 +1,4 @@ +%% @doc Cluster join logic. -module(cets_join). -export([join/4]). -include_lib("kernel/include/logger.hrl"). From 68a65470f9ab2d185467dc8eb6bb9961a924e179 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Sat, 22 Oct 2022 23:50:31 +0200 Subject: [PATCH 53/64] Fix dialyzer --- src/cets.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cets.erl b/src/cets.erl index 980e74b5..269e8ab7 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -51,7 +51,8 @@ pause_monitors := [pause_monitor()]}. -type long_msg() :: pause | ping | remote_dump | sync | table_name | get_info - | other_servers | {unpause, reference()}. + | other_servers | {unpause, reference()} + | {send_dump, [pid()], [tuple()]}. -type info() :: #{table := table_name(), nodes := [node()], From 65cf64771a9496e941edd43cb7da53ae0d8941fe Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 6 Mar 2023 20:43:10 +0100 Subject: [PATCH 54/64] Autoformat with erlfmt --- .github/workflows/ci.yml | 11 +++ rebar.config | 26 +++-- src/cets.app.src | 18 ++-- src/cets.erl | 185 +++++++++++++++++++++++++----------- src/cets_call.erl | 11 ++- src/cets_discovery.erl | 37 +++++--- src/cets_discovery_file.erl | 9 +- src/cets_join.erl | 21 ++-- src/cets_long.erl | 40 ++++---- src/cets_mon_cleaner.erl | 30 ++++-- test/cets_SUITE.erl | 66 +++++++------ 11 files changed, 299 insertions(+), 155 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ed05840..6b856539 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,3 +48,14 @@ jobs: run: rebar3 compile - name: Run xref run: rebar3 xref + + erlfmt: + runs-on: ubuntu-latest + + container: + image: erlang:23 + + steps: + - uses: actions/checkout@v3 + - name: Run erlfmt + run: rebar3 fmt --check diff --git a/rebar.config b/rebar.config index c2f77ada..fe4f279f 100644 --- a/rebar.config +++ b/rebar.config @@ -1,9 +1,21 @@ {dialyzer, [ - {warnings, - [no_return, no_unused, no_improper_lists, no_fun_app, no_match, - no_opaque, no_fail_call, no_contracts, no_behaviours, - no_undefined_callbacks, unmatched_returns, error_handling, - underspecs - % overspecs, specdiffs - ]}]}. + {warnings, [ + no_return, + no_unused, + no_improper_lists, + no_fun_app, + no_match, + no_opaque, + no_fail_call, + no_contracts, + no_behaviours, + no_undefined_callbacks, + unmatched_returns, + error_handling, + underspecs + % overspecs, specdiffs + ]} +]}. +%% Enables "rebar3 fmt" command +{project_plugins, [erlfmt]}. diff --git a/src/cets.app.src b/src/cets.app.src index fced7441..00ae2f22 100644 --- a/src/cets.app.src +++ b/src/cets.app.src @@ -1,9 +1,9 @@ -{application, cets, - [{description, "Clustered Erlang Term Storage"}, - {vsn, "0.1"}, - {modules, []}, - {registered, []}, - {applications, [kernel, stdlib]}, - {env, []} -% {mod, {cets_app, []} - ]}. +{application, cets, [ + {description, "Clustered Erlang Term Storage"}, + {vsn, "0.1"}, + {modules, []}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []} + % {mod, {cets_app, []} +]}. diff --git a/src/cets.erl b/src/cets.erl index 269e8ab7..28187ee6 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -17,47 +17,101 @@ -module(cets). -behaviour(gen_server). --export([start/2, stop/1, insert/2, insert_many/2, delete/2, delete_many/2, - dump/1, remote_dump/1, send_dump/3, table_name/1, - other_nodes/1, other_pids/1, - pause/1, unpause/2, sync/1, ping/1, info/1, - insert_request/2, insert_many_request/2, - delete_request/2, delete_many_request/2, wait_response/2, - init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --ignore_xref([start/2, stop/1, insert/2, insert_many/2, delete/2, delete_many/2, - pause/1, unpause/2, sync/1, ping/1, info/1, other_nodes/1, - insert_request/2, insert_many_request/2, - delete_request/2, delete_many_request/2, wait_response/2]). +-export([ + start/2, + stop/1, + insert/2, + insert_many/2, + delete/2, + delete_many/2, + dump/1, + remote_dump/1, + send_dump/3, + table_name/1, + other_nodes/1, + other_pids/1, + pause/1, + unpause/2, + sync/1, + ping/1, + info/1, + insert_request/2, + insert_many_request/2, + delete_request/2, + delete_many_request/2, + wait_response/2, + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +-ignore_xref([ + start/2, + stop/1, + insert/2, + insert_many/2, + delete/2, + delete_many/2, + pause/1, + unpause/2, + sync/1, + ping/1, + info/1, + other_nodes/1, + insert_request/2, + insert_many_request/2, + delete_request/2, + delete_many_request/2, + wait_response/2 +]). -include_lib("kernel/include/logger.hrl"). --type server_ref() :: pid() | atom() | {local, atom()} - | {global, term()} | {via, module(), term()}. +-type server_ref() :: + pid() + | atom() + | {local, atom()} + | {global, term()} + | {via, module(), term()}. -type request_id() :: reference(). --type op() :: {insert, tuple()} | {delete, term()} - | {insert_many, [tuple()]} | {delete_many, [term()]}. +-type op() :: + {insert, tuple()} + | {delete, term()} + | {insert_many, [tuple()]} + | {delete_many, [term()]}. -type from() :: {pid(), reference()}. -type backlog_entry() :: {op(), from()}. -type table_name() :: atom(). -type pause_monitor() :: reference(). -type state() :: #{ - tab := table_name(), - mon_tab := atom(), - other_servers := [pid()], - opts := start_opts(), - backlog := [backlog_entry()], - pause_monitors := [pause_monitor()]}. - --type long_msg() :: pause | ping | remote_dump | sync | table_name | get_info - | other_servers | {unpause, reference()} - | {send_dump, [pid()], [tuple()]}. - --type info() :: #{table := table_name(), - nodes := [node()], - size := non_neg_integer(), - memory := non_neg_integer()}. + tab := table_name(), + mon_tab := atom(), + other_servers := [pid()], + opts := start_opts(), + backlog := [backlog_entry()], + pause_monitors := [pause_monitor()] +}. + +-type long_msg() :: + pause + | ping + | remote_dump + | sync + | table_name + | get_info + | other_servers + | {unpause, reference()} + | {send_dump, [pid()], [tuple()]}. + +-type info() :: #{ + table := table_name(), + nodes := [node()], + size := non_neg_integer(), + memory := non_neg_integer() +}. -type handle_down_fun() :: fun((#{remote_pid := pid(), table := table_name()}) -> ok). -type start_opts() :: #{handle_down := handle_down_fun()}. @@ -187,20 +241,25 @@ init({Tab, Opts}) -> _ = ets:new(Tab, [ordered_set, named_table, public]), _ = ets:new(MonTab, [public, named_table]), {ok, _} = cets_mon_cleaner:start_link(MonTab, MonTab), - {ok, #{tab => Tab, mon_tab => MonTab, - other_servers => [], opts => Opts, backlog => [], - pause_monitors => []}}. + {ok, #{ + tab => Tab, + mon_tab => MonTab, + other_servers => [], + opts => Opts, + backlog => [], + pause_monitors => [] + }}. -spec handle_call(term(), from(), state()) -> - {noreply, state()} | {reply, term(), state()}. + {noreply, state()} | {reply, term(), state()}. handle_call(other_servers, _From, State = #{other_servers := Servers}) -> {reply, Servers, State}; handle_call(sync, From, State = #{other_servers := Servers}) -> %% Do spawn to avoid any possible deadlocks proc_lib:spawn(fun() -> - lists:foreach(fun ping/1, Servers), - gen_server:reply(From, ok) - end), + lists:foreach(fun ping/1, Servers), + gen_server:reply(From, ok) + end), {noreply, State}; handle_call(ping, _From, State) -> {reply, pong, State}; @@ -215,7 +274,7 @@ handle_call({send_dump, NewPids, Dump}, _From, State) -> handle_call(pause, _From = {FromPid, _}, State = #{pause_monitors := Mons}) -> %% We monitor who pauses our server Mon = erlang:monitor(process, FromPid), - {reply, Mon, State#{pause_monitors := [Mon|Mons]}}; + {reply, Mon, State#{pause_monitors := [Mon | Mons]}}; handle_call({unpause, Ref}, _From, State) -> handle_unpause(Ref, State); handle_call(get_info, _From, State) -> @@ -225,7 +284,7 @@ handle_call(get_info, _From, State) -> handle_cast({op, From, Msg}, State = #{pause_monitors := []}) -> handle_op(From, Msg, State), {noreply, State}; -handle_cast({op, From, Msg}, State = #{pause_monitors := [_|_], backlog := Backlog}) -> +handle_cast({op, From, Msg}, State = #{pause_monitors := [_ | _], backlog := Backlog}) -> %% Backlog is a list of pending operation, when our server is paused. %% The list would be applied, once our server is unpaused. {noreply, State#{backlog := [{Msg, From} | Backlog]}}; @@ -259,8 +318,11 @@ handle_send_dump(NewPids, Dump, State = #{tab := Tab, other_servers := Servers}) handle_down(Mon, Pid, State = #{pause_monitors := Mons}) -> case lists:member(Mon, Mons) of true -> - ?LOG_ERROR(#{what => pause_owner_crashed, - state => State, paused_by_pid => Pid}), + ?LOG_ERROR(#{ + what => pause_owner_crashed, + state => State, + paused_by_pid => Pid + }), {reply, ok, State2} = handle_unpause(Mon, State), {noreply, State2}; false -> @@ -276,8 +338,11 @@ handle_down2(_Mon, RemotePid, State = #{other_servers := Servers, mon_tab := Mon {noreply, State#{other_servers := Servers2}}; false -> %% This should not happen - ?LOG_ERROR(#{what => handle_down_failed, - remote_pid => RemotePid, state => State}), + ?LOG_ERROR(#{ + what => handle_down_failed, + remote_pid => RemotePid, + state => State + }), {noreply, State} end. @@ -295,15 +360,19 @@ notify_remote_down_loop(_RemotePid, []) -> add_servers(Pids, Servers) -> lists:sort(add_servers2(Pids, Servers) ++ Servers). -add_servers2([RemotePid | OtherPids], Servers) - when is_pid(RemotePid), RemotePid =/= self() -> +add_servers2([RemotePid | OtherPids], Servers) when + is_pid(RemotePid), RemotePid =/= self() +-> case has_remote_pid(RemotePid, Servers) of false -> erlang:monitor(process, RemotePid), [RemotePid | add_servers2(OtherPids, Servers)]; true -> - ?LOG_INFO(#{what => already_added, - remote_pid => RemotePid, remote_node => node(RemotePid)}), + ?LOG_INFO(#{ + what => already_added, + remote_pid => RemotePid, + remote_node => node(RemotePid) + }), add_servers2(OtherPids, Servers) end; add_servers2([], _Servers) -> @@ -401,10 +470,12 @@ handle_unpause2(Mon, Mons, State) -> -spec handle_get_info(state()) -> {reply, info(), state()}. handle_get_info(State = #{tab := Tab, other_servers := Servers}) -> - Info = #{table => Tab, - nodes => lists:usort(pids_to_nodes([self() | Servers])), - size => ets:info(Tab, size), - memory => ets:info(Tab, memory)}, + Info = #{ + table => Tab, + nodes => lists:usort(pids_to_nodes([self() | Servers])), + size => ets:info(Tab, size), + memory => ets:info(Tab, memory) + }, {reply, Info, State}. %% Cleanup @@ -412,8 +483,12 @@ call_user_handle_down(RemotePid, _State = #{tab := Tab, opts := Opts}) -> case Opts of #{handle_down := F} -> FF = fun() -> F(#{remote_pid => RemotePid, table => Tab}) end, - Info = #{task => call_user_handle_down, table => Tab, - remote_pid => RemotePid, remote_node => node(RemotePid)}, + Info = #{ + task => call_user_handle_down, + table => Tab, + remote_pid => RemotePid, + remote_node => node(RemotePid) + }, cets_long:run_safely(Info, FF); _ -> ok diff --git a/src/cets_call.erl b/src/cets_call.erl index 11127c38..08b500ca 100644 --- a/src/cets_call.erl +++ b/src/cets_call.erl @@ -24,8 +24,11 @@ long_call(Server, Msg) -> long_call(Server, Msg, Info) -> case where(Server) of Pid when is_pid(Pid) -> - Info2 = Info#{remote_server => Server, remote_pid => Pid, - remote_node => node(Pid)}, + Info2 = Info#{ + remote_server => Server, + remote_pid => Pid, + remote_node => node(Pid) + }, F = fun() -> gen_server:call(Pid, Msg, infinity) end, cets_long:run_safely(Info2, F); undefined -> @@ -69,8 +72,8 @@ wait_response(Mon, Timeout) -> {cets_reply, Mon, WaitInfo} -> wait_for_updated(Mon, WaitInfo) after Timeout -> - erlang:demonitor(Mon, [flush]), - error(timeout) + erlang:demonitor(Mon, [flush]), + error(timeout) end. %% Wait for response from the remote nodes that the operation is completed. diff --git a/src/cets_discovery.erl b/src/cets_discovery.erl index 45a586eb..98da8aa8 100644 --- a/src/cets_discovery.erl +++ b/src/cets_discovery.erl @@ -4,8 +4,14 @@ -behaviour(gen_server). -export([start/1, start_link/1, add_table/2, info/1]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). -ignore_xref([start/1, start_link/1, add_table/2, info/1, behaviour_info/1]). @@ -16,12 +22,12 @@ -type from() :: {pid(), reference()}. -type state() :: #{ - results := [term()], - tables := [atom()], - backend_module := module(), - backend_state := state(), - timer_ref := reference() | undefined - }. + results := [term()], + tables := [atom()], + backend_module := module(), + backend_state := state(), + timer_ref := reference() | undefined +}. -callback init(map()) -> backend_state(). -callback get_nodes(backend_state()) -> {get_nodes_result(), backend_state()}. @@ -58,9 +64,13 @@ init(Opts) -> self() ! check, Tables = maps:get(tables, Opts, []), BackendState = Mod:init(Opts), - {ok, #{results => [], tables => Tables, - backend_module => Mod, backend_state => BackendState, - timer_ref => undefined}}. + {ok, #{ + results => [], + tables => Tables, + backend_module => Mod, + backend_state => BackendState, + timer_ref => undefined + }}. -spec handle_call(term(), from(), state()) -> {reply, term(), state()}. handle_call({add_table, Table}, _From, State = #{tables := Tables}) -> @@ -124,7 +134,10 @@ cancel_old_timer(_State) -> ok. flush_all_checks() -> - receive check -> flush_all_checks() after 0 -> ok end. + receive + check -> flush_all_checks() + after 0 -> ok + end. do_join(Tab, Node) -> LocalPid = whereis(Tab), diff --git a/src/cets_discovery_file.erl b/src/cets_discovery_file.erl index 198623c4..1ca714fc 100644 --- a/src/cets_discovery_file.erl +++ b/src/cets_discovery_file.erl @@ -1,5 +1,5 @@ %% @doc File backend for cets_discovery. -%% +%% %% Barebone AWS EC2 auto-discovery is limited: %% - UDP broadcasts do not work %% - AWS CLI needs access @@ -18,8 +18,11 @@ init(Opts) -> get_nodes(State = #{disco_file := Filename}) -> case file:read_file(Filename) of {error, Reason} -> - ?LOG_ERROR(#{what => discovery_failed, - filename => Filename, reason => Reason}), + ?LOG_ERROR(#{ + what => discovery_failed, + filename => Filename, + reason => Reason + }), {{error, Reason}, State}; {ok, Text} -> Lines = binary:split(Text, [<<"\r">>, <<"\n">>, <<" ">>], [global]), diff --git a/src/cets_join.erl b/src/cets_join.erl index 3e400a84..f22d1914 100644 --- a/src/cets_join.erl +++ b/src/cets_join.erl @@ -7,8 +7,11 @@ %% Writes from other nodes would wait for join completion. %% LockKey should be the same on all nodes. join(LockKey, Info, LocalPid, RemotePid) when is_pid(LocalPid), is_pid(RemotePid) -> - Info2 = Info#{local_pid => LocalPid, - remote_pid => RemotePid, remote_node => node(RemotePid)}, + Info2 = Info#{ + local_pid => LocalPid, + remote_pid => RemotePid, + remote_node => node(RemotePid) + }, F = fun() -> join1(LockKey, Info2, LocalPid, RemotePid) end, cets_long:run_safely(Info2#{what => join_failed}, F). @@ -18,9 +21,9 @@ join1(LockKey, Info, LocalPid, RemotePid) -> true -> {error, already_joined}; false -> - Start = erlang:system_time(millisecond), - F = fun() -> join_loop(LockKey, Info, LocalPid, RemotePid, Start) end, - cets_long:run(Info#{task => join}, F) + Start = erlang:system_time(millisecond), + F = fun() -> join_loop(LockKey, Info, LocalPid, RemotePid, Start) end, + cets_long:run(Info#{task => join}, F) end. join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> @@ -34,7 +37,7 @@ join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> ?LOG_INFO(Info#{what => join_got_lock, after_time_ms => Diff}), %% Do joining in a separate process to reduce GC cets_long:run_spawn(Info, fun() -> join2(Info, LocalPid, RemotePid) end) - end, + end, LockRequest = {LockKey, self()}, %% Just lock all nodes, no magic here :) Nodes = [node() | nodes()], @@ -73,6 +76,8 @@ join2(_Info, LocalPid, RemotePid) -> remote_or_local_dump(Pid) when node(Pid) =:= node() -> {ok, Tab} = cets:table_name(Pid), - {ok, cets:dump(Tab)}; %% Reduce copying + %% Reduce copying + {ok, cets:dump(Tab)}; remote_or_local_dump(Pid) -> - cets:remote_dump(Pid). %% We actually need to ask the remote process + %% We actually need to ask the remote process + cets:remote_dump(Pid). diff --git a/src/cets_long.erl b/src/cets_long.erl index 17ee0eab..22d180c2 100644 --- a/src/cets_long.erl +++ b/src/cets_long.erl @@ -12,9 +12,9 @@ run_spawn(Info, F) -> Pid = self(), Ref = make_ref(), proc_lib:spawn_link(fun() -> - Res = cets_long:run_safely(Info, F), - Pid ! {result, Ref, Res} - end), + Res = cets_long:run_safely(Info, F), + Pid ! {result, Ref, Res} + end), receive {result, Ref, Res} -> Res @@ -32,14 +32,14 @@ run(Info, Fun, Catch) -> ?LOG_INFO(Info#{what => long_task_started}), Pid = spawn_mon(Info, Parent, Start), try - case Catch of - true -> just_run_safely(Info#{what => long_task_failed}, Fun); - false -> Fun() - end - after - Diff = diff(Start), - ?LOG_INFO(Info#{what => long_task_finished, time_ms => Diff}), - Pid ! stop + case Catch of + true -> just_run_safely(Info#{what => long_task_failed}, Fun); + false -> Fun() + end + after + Diff = diff(Start), + ?LOG_INFO(Info#{what => long_task_finished, time_ms => Diff}), + Pid ! stop end. spawn_mon(Info, Parent, Start) -> @@ -54,11 +54,12 @@ monitor_loop(Mon, Info, Start) -> {'DOWN', MonRef, process, _Pid, Reason} when Mon =:= MonRef -> ?LOG_ERROR(Info#{what => long_task_failed, reason => Reason}), ok; - stop -> ok - after 5000 -> - Diff = diff(Start), - ?LOG_INFO(Info#{what => long_task_progress, time_ms => Diff}), - monitor_loop(Mon, Info, Start) + stop -> + ok + after 5000 -> + Diff = diff(Start), + ?LOG_INFO(Info#{what => long_task_progress, time_ms => Diff}), + monitor_loop(Mon, Info, Start) end. diff(Start) -> @@ -67,7 +68,8 @@ diff(Start) -> just_run_safely(Info, Fun) -> try Fun() - catch Class:Reason:Stacktrace -> - ?LOG_ERROR(Info#{class => Class, reason => Reason, stacktrace => Stacktrace}), - {error, {Class, Reason, Stacktrace}} + catch + Class:Reason:Stacktrace -> + ?LOG_ERROR(Info#{class => Class, reason => Reason, stacktrace => Stacktrace}), + {error, {Class, Reason, Stacktrace}} end. diff --git a/src/cets_mon_cleaner.erl b/src/cets_mon_cleaner.erl index 860a9971..3fb64f16 100644 --- a/src/cets_mon_cleaner.erl +++ b/src/cets_mon_cleaner.erl @@ -7,17 +7,23 @@ -behaviour(gen_server). -export([start_link/2]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). -include_lib("kernel/include/logger.hrl"). -type timer_ref() :: reference(). -type state() :: #{ - mon_tab := atom(), - interval := non_neg_integer(), - timer_ref := timer_ref() - }. + mon_tab := atom(), + interval := non_neg_integer(), + timer_ref := timer_ref() +}. start_link(Name, MonTab) -> gen_server:start_link({local, Name}, ?MODULE, MonTab, []). @@ -25,8 +31,11 @@ start_link(Name, MonTab) -> -spec init(atom()) -> {ok, state()}. init(MonTab) -> Interval = 30000, - State = #{mon_tab => MonTab, interval => Interval, - timer_ref => start_timer(Interval)}, + State = #{ + mon_tab => MonTab, + interval => Interval, + timer_ref => start_timer(Interval) + }, {ok, State}. handle_call(Msg, From, State) -> @@ -56,7 +65,10 @@ schedule_check(State = #{interval := Interval, timer_ref := OldRef}) -> State#{timer_ref := start_timer(Interval)}. flush_all_checks() -> - receive check -> flush_all_checks() after 0 -> ok end. + receive + check -> flush_all_checks() + after 0 -> ok + end. start_timer(Interval) -> erlang:send_after(Interval, self(), check). diff --git a/test/cets_SUITE.erl b/test/cets_SUITE.erl index d8d4b82d..c7143fdd 100644 --- a/test/cets_SUITE.erl +++ b/test/cets_SUITE.erl @@ -1,23 +1,29 @@ -module(cets_SUITE). -include_lib("common_test/include/ct.hrl"). - + -compile([export_all, nowarn_export_all]). - -all() -> [test_multinode, node_list_is_correct, - test_multinode_auto_discovery, test_locally, - handle_down_is_called, - events_are_applied_in_the_correct_order_after_unpause, - pause_multiple_times, - unpause_twice, - write_returns_if_remote_server_crashes, - mon_cleaner_works, sync_using_name_works, - insert_many_request]. - + +all() -> + [ + test_multinode, + node_list_is_correct, + test_multinode_auto_discovery, + test_locally, + handle_down_is_called, + events_are_applied_in_the_correct_order_after_unpause, + pause_multiple_times, + unpause_twice, + write_returns_if_remote_server_crashes, + mon_cleaner_works, + sync_using_name_works, + insert_many_request + ]. + init_per_suite(Config) -> Node2 = start_node(ct2), Node3 = start_node(ct3), Node4 = start_node(ct4), - [{nodes, [Node2, Node3, Node4]}|Config]. + [{nodes, [Node2, Node3, Node4]} | Config]. end_per_suite(Config) -> Config. @@ -27,10 +33,10 @@ init_per_testcase(test_multinode_auto_discovery, Config) -> Config; init_per_testcase(_, Config) -> Config. - + end_per_testcase(_, _Config) -> ok. - + test_multinode(Config) -> Node1 = node(), [Node2, Node3, Node4] = proplists:get_value(nodes, Config), @@ -53,12 +59,12 @@ test_multinode(Config) -> insert(Node1, Tab, {f}), insert(Node4, Tab, {e}), Same = fun(X) -> - X = dump(Node1, Tab), - X = dump(Node2, Tab), - X = dump(Node3, Tab), - X = dump(Node4, Tab), - ok - end, + X = dump(Node1, Tab), + X = dump(Node2, Tab), + X = dump(Node3, Tab), + X = dump(Node4, Tab), + ok + end, Same([{a}, {b}, {c}, {d}, {e}, {f}]), delete(Node1, Tab, e), Same([{a}, {b}, {c}, {d}, {f}]), @@ -67,7 +73,7 @@ test_multinode(Config) -> %% Bulk operations are supported insert_many(Node4, Tab, [{m}, {a}, {n}, {y}]), Same([{a}, {b}, {c}, {d}, {f}, {m}, {n}, {y}]), - delete_many(Node4, Tab, [a,n]), + delete_many(Node4, Tab, [a, n]), Same([{b}, {c}, {d}, {f}, {m}, {y}]), ok. @@ -102,8 +108,8 @@ test_multinode_auto_discovery(Config) -> %% Waits for the first check sys:get_state(Disco), [Node2] = other_nodes(Node1, Tab), - [#{memory := _, nodes := [Node1, Node2], size := 0, table := tab2}] - = cets_discovery:info(Disco), + [#{memory := _, nodes := [Node1, Node2], size := 0, table := tab2}] = + cets_discovery:info(Disco), ok. test_locally(_Config) -> @@ -119,8 +125,8 @@ test_locally(_Config) -> handle_down_is_called(_Config) -> Parent = self(), DownFn = fun(#{remote_pid := _RemotePid, table := _Tab}) -> - Parent ! down_called - end, + Parent ! down_called + end, {ok, Pid1} = cets:start(d1, #{handle_down => DownFn}), {ok, Pid2} = cets:start(d2, #{}), ok = cets_join:join(lock1, #{table => [d1, d2]}, Pid1, Pid2), @@ -155,10 +161,12 @@ pause_multiple_times(_Config) -> PauseMon2 = cets:pause(Pid), Ref1 = cets:insert_request(Pid, {1}), Ref2 = cets:insert_request(Pid, {2}), - [] = cets:dump(T), %% No records yet, even after pong + %% No records yet, even after pong + [] = cets:dump(T), ok = cets:unpause(Pid, PauseMon1), pong = cets:ping(Pid), - [] = cets:dump(T), %% No records yet, even after pong + %% No records yet, even after pong + [] = cets:dump(T), ok = cets:unpause(Pid, PauseMon2), pong = cets:ping(Pid), cets:wait_response(Ref1, 5000), @@ -185,7 +193,7 @@ mon_cleaner_works(_Config) -> {ok, Pid1} = cets:start(c3, #{}), %% Suspend, so to avoid unexpected check sys:suspend(c3_mon), - %% Two cases to check: an alive process and a dead process + %% Two cases to check: an alive process and a dead process R = cets:insert_request(c3, {2}), %% Ensure insert_request reaches the server cets:ping(Pid1), From 2e5c835db3a513b2794fc96d01db13922bfb605c Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 6 Mar 2023 20:54:45 +0100 Subject: [PATCH 55/64] Update erlang version to 25 --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b856539..c422963a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest container: - image: erlang:23 + image: erlang:25 steps: - uses: actions/checkout@v3 @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest container: - image: erlang:23 + image: erlang:25 steps: - uses: actions/checkout@v3 @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest container: - image: erlang:23 + image: erlang:25 steps: - uses: actions/checkout@v3 @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest container: - image: erlang:23 + image: erlang:25 steps: - uses: actions/checkout@v3 From be8c904d9654af864afbf54d0190134ada9c2f80 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 6 Mar 2023 21:39:43 +0100 Subject: [PATCH 56/64] Add specs for cets_discovery Address some review comments --- .gitignore | 1 - README.md | 8 ++++---- src/cets.erl | 22 +++++----------------- src/cets_call.erl | 14 +++++++------- src/cets_discovery.erl | 12 ++++++++++++ src/cets_discovery_file.erl | 5 +++++ 6 files changed, 33 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index 9ccc8e03..add99d19 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ _build/ -rebar3 # vim temporary files *.sw* diff --git a/README.md b/README.md index fdc2dd7b..07a035bf 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,9 @@ It exports functions: - `delete(Server, Key)` - deletes an object from the table. - `delete_many(Server, Keys)` - deletes several objects from the table. -cets_join module contains the merging logic. +`cets_join` module contains the merging logic. -cets_discovery module handles search of new nodes. +`cets_discovery` module handles search of new nodes. It supports behaviours for different backends. @@ -47,9 +47,9 @@ It defines two callbacks: - `init/1` - inits the backend. - `get_nodes/1` - gets a list of alive erlang nodes. -Once new nodes are found, cets_discovery calls cets_join module to merge two +Once new nodes are found, `cets_discovery` calls `cets_join` module to merge two cluster partitions. -The simplest cets_discovery backend is cets_discovery_file, which just reads +The simplest `cets_discovery` backend is `cets_discovery_file`, which just reads a file with a list of nodes on each line. This file could be populated by an external program or by an admin. diff --git a/src/cets.erl b/src/cets.erl index 28187ee6..72ce65ee 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -116,13 +116,13 @@ -type handle_down_fun() :: fun((#{remote_pid := pid(), table := table_name()}) -> ok). -type start_opts() :: #{handle_down := handle_down_fun()}. --export_type([request_id/0, op/0, server_ref/0, long_msg/0]). +-export_type([request_id/0, op/0, server_ref/0, long_msg/0, info/0, table_name/0]). %% API functions %% Table and server has the same name %% Opts: -%% - handle_down = fun(#{remote_pid => Pid, table => Tab}) +%% - handle_down = fun(#{remote_pid := Pid, table := Tab}) %% Called when a remote node goes down. Do not update other nodes data %% from this function (otherwise circular locking could happen - use spawn %% to make a new async process if you need to update). @@ -136,7 +136,7 @@ start(Tab, Opts) when is_atom(Tab) -> stop(Server) -> gen_server:stop(Server). --spec dump(server_ref()) -> Records :: [tuple()]. +-spec dump(table_name()) -> Records :: [tuple()]. dump(Tab) -> ets:tab2list(Tab). @@ -425,26 +425,14 @@ handle_op(From = {Mon, Pid}, Msg, State) when is_pid(Pid) -> replicate(From, Msg, #{mon_tab := MonTab, other_servers := Servers}) -> %% Reply would be routed directly to FromPid Msg2 = {remote_op, From, Msg}, - replicate2(Servers, Msg2), + [send_to_remote(RemotePid, Msg2) || RemotePid <- Servers], ets:insert(MonTab, From), {Servers, MonTab}. -replicate2([RemotePid | Servers], Msg) -> - send_to_remote(RemotePid, Msg), - replicate2(Servers, Msg); -replicate2([], _Msg) -> - ok. - apply_backlog(State = #{backlog := Backlog}) -> - apply_backlog_ops(lists:reverse(Backlog), State), + [handle_op(From, Msg, State) || {Msg, From} <- lists:reverse(Backlog)], State#{backlog := []}. -apply_backlog_ops([{Msg, From} | Backlog], State) -> - handle_op(From, Msg, State), - apply_backlog_ops(Backlog, State); -apply_backlog_ops([], _State) -> - ok. - %% We support multiple pauses %% Only when all pause requests are unpaused we continue handle_unpause(Mon, State = #{pause_monitors := Mons}) -> diff --git a/src/cets_call.erl b/src/cets_call.erl index 08b500ca..322bce07 100644 --- a/src/cets_call.erl +++ b/src/cets_call.erl @@ -14,8 +14,8 @@ -type server_ref() :: cets:server_ref(). -type long_msg() :: cets:long_msg(). -%% Do gen_server:call with better error reporting. -%% It would print a warning if the call takes too long. +%% Does gen_server:call with better error reporting. +%% It would log a warning if the call takes too long. -spec long_call(server_ref(), long_msg()) -> term(). long_call(Server, Msg) -> long_call(Server, Msg, #{msg => Msg}). @@ -80,25 +80,25 @@ wait_response(Mon, Timeout) -> %% remote_down is sent by the local server, if the remote server is down. wait_for_updated(Mon, {Servers, MonTab}) -> try - wait_for_updated2(Mon, Servers) + do_wait_for_updated(Mon, Servers) after erlang:demonitor(Mon, [flush]), ets:delete(MonTab, Mon) end. -wait_for_updated2(_Mon, []) -> +do_wait_for_updated(_Mon, []) -> ok; -wait_for_updated2(Mon, Servers) -> +do_wait_for_updated(Mon, Servers) -> receive {updated, Mon, Pid} -> %% A replication confirmation from the remote server is received Servers2 = lists:delete(Pid, Servers), - wait_for_updated2(Mon, Servers2); + do_wait_for_updated(Mon, Servers2); {remote_down, Mon, Pid} -> %% This message is sent by our local server when %% the remote server is down condition is detected Servers2 = lists:delete(Pid, Servers), - wait_for_updated2(Mon, Servers2); + do_wait_for_updated(Mon, Servers2); {'DOWN', Mon, process, _Pid, Reason} -> %% Local server is down, this is a critical error error({cets_down, Reason}) diff --git a/src/cets_discovery.erl b/src/cets_discovery.erl index 98da8aa8..1fef9d31 100644 --- a/src/cets_discovery.erl +++ b/src/cets_discovery.erl @@ -20,6 +20,8 @@ -type backend_state() :: term(). -type get_nodes_result() :: {ok, [node()]} | {error, term()}. +-export_type([get_nodes_result/0]). + -type from() :: {pid(), reference()}. -type state() :: #{ results := [term()], @@ -29,12 +31,19 @@ timer_ref := reference() | undefined }. +%% Backend could define its own options +-type opts() :: #{name := atom(), _ := _}. +-type start_result() :: {ok, pid()} | {error, term()}. +-type server() :: pid() | atom(). + -callback init(map()) -> backend_state(). -callback get_nodes(backend_state()) -> {get_nodes_result(), backend_state()}. +-spec start(opts()) -> start_result(). start(Opts) -> start_common(start, Opts). +-spec start_link(opts()) -> start_result(). start_link(Opts) -> start_common(start_link, Opts). @@ -48,12 +57,15 @@ start_common(F, Opts) -> end, apply(gen_server, F, Args). +-spec add_table(server(), cets:table_name()) -> ok | {error, already_added}. add_table(Server, Table) -> gen_server:call(Server, {add_table, Table}). +-spec get_tables(server()) -> [cets:table_name()]. get_tables(Server) -> gen_server:call(Server, get_tables). +-spec info(server()) -> [cets:info()]. info(Server) -> {ok, Tables} = get_tables(Server), [cets:info(Tab) || Tab <- Tables]. diff --git a/src/cets_discovery_file.erl b/src/cets_discovery_file.erl index 1ca714fc..3e7aeb6f 100644 --- a/src/cets_discovery_file.erl +++ b/src/cets_discovery_file.erl @@ -12,9 +12,14 @@ -include_lib("kernel/include/logger.hrl"). +-type opts() :: #{disco_file := file:filename()}. +-type state() :: opts(). + +-spec init(opts()) -> state(). init(Opts) -> Opts. +-spec get_nodes(state()) -> {cets_discovery:get_nodes_result(), state()}. get_nodes(State = #{disco_file := Filename}) -> case file:read_file(Filename) of {error, Reason} -> From dab33e572b50f66f26e08c526f0390237566192d Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 9 Mar 2023 10:43:30 +0100 Subject: [PATCH 57/64] Add more granular tests Test each operation of test_multinode separately --- run_test.sh | 2 +- test/cets_SUITE.erl | 70 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/run_test.sh b/run_test.sh index 9a214163..a4779fc6 100755 --- a/run_test.sh +++ b/run_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -./rebar3 ct --sname=ct1 +rebar3 ct --sname=ct1 diff --git a/test/cets_SUITE.erl b/test/cets_SUITE.erl index c7143fdd..f6e32492 100644 --- a/test/cets_SUITE.erl +++ b/test/cets_SUITE.erl @@ -5,6 +5,15 @@ all() -> [ + inserted_records_could_be_read_back, + insert_many_with_one_record, + insert_many_with_two_records, + delete_works, + delete_many_works, + join_works, + inserted_records_could_be_read_back_from_replicated_table, + join_works_with_existing_data, + join_works_with_existing_data_with_conflicts, test_multinode, node_list_is_correct, test_multinode_auto_discovery, @@ -37,6 +46,67 @@ init_per_testcase(_, Config) -> end_per_testcase(_, _Config) -> ok. +inserted_records_could_be_read_back(_Config) -> + cets:start(ins1, #{}), + cets:insert(ins1, {alice, 32}), + [{alice, 32}] = ets:lookup(ins1, alice). + +insert_many_with_one_record(_Config) -> + cets:start(ins1m, #{}), + cets:insert_many(ins1m, [{alice, 32}]), + [{alice, 32}] = ets:lookup(ins1m, alice). + +insert_many_with_two_records(_Config) -> + cets:start(ins2m, #{}), + cets:insert_many(ins2m, [{alice, 32}, {bob, 55}]), + [{alice, 32}, {bob, 55}] = ets:tab2list(ins2m). + +delete_works(_Config) -> + cets:start(del1, #{}), + cets:insert(del1, {alice, 32}), + cets:delete(del1, alice), + [] = ets:lookup(del1, alice). + +delete_many_works(_Config) -> + cets:start(del1, #{}), + cets:insert(del1, {alice, 32}), + cets:delete_many(del1, [alice]), + [] = ets:lookup(del1, alice). + +join_works(_Config) -> + {ok, Pid1} = cets:start(join1tab, #{}), + {ok, Pid2} = cets:start(join2tab, #{}), + ok = cets_join:join(join_lock1, #{}, Pid1, Pid2). + +inserted_records_could_be_read_back_from_replicated_table(_Config) -> + {ok, Pid1} = cets:start(ins1tab, #{}), + {ok, Pid2} = cets:start(ins2tab, #{}), + ok = cets_join:join(join_lock1_ins, #{}, Pid1, Pid2), + cets:insert(ins1tab, {alice, 32}), + [{alice, 32}] = ets:lookup(ins2tab, alice). + +join_works_with_existing_data(_Config) -> + {ok, Pid1} = cets:start(ex1tab, #{}), + {ok, Pid2} = cets:start(ex2tab, #{}), + cets:insert(ex1tab, {alice, 32}), + %% Join will copy and merge existing tables + ok = cets_join:join(join_lock1_ex, #{}, Pid1, Pid2), + [{alice, 32}] = ets:lookup(ex2tab, alice). + +%% This testcase tests an edgecase: inserting with the same key from two nodes. +%% Usually, inserting with the same key from two different nodes is not possible +%% (because the node-name is a part of the key). +join_works_with_existing_data_with_conflicts(_Config) -> + {ok, Pid1} = cets:start(con1tab, #{}), + {ok, Pid2} = cets:start(con2tab, #{}), + cets:insert(con1tab, {alice, 32}), + cets:insert(con2tab, {alice, 33}), + %% Join will copy and merge existing tables + ok = cets_join:join(join_lock1_con, #{}, Pid1, Pid2), + %% We insert data from other table into our table when merging, so the values get swapped + [{alice, 33}] = ets:lookup(con1tab, alice), + [{alice, 32}] = ets:lookup(con2tab, alice). + test_multinode(Config) -> Node1 = node(), [Node2, Node3, Node4] = proplists:get_value(nodes, Config), From 7dcb8d09a7d1cdfc346c9a0484bdba580b6207ad Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 13 Mar 2023 09:32:15 +0100 Subject: [PATCH 58/64] Remove unnecessary cets_long:run call in join --- src/cets_join.erl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cets_join.erl b/src/cets_join.erl index f22d1914..44e4497c 100644 --- a/src/cets_join.erl +++ b/src/cets_join.erl @@ -13,7 +13,7 @@ join(LockKey, Info, LocalPid, RemotePid) when is_pid(LocalPid), is_pid(RemotePid remote_node => node(RemotePid) }, F = fun() -> join1(LockKey, Info2, LocalPid, RemotePid) end, - cets_long:run_safely(Info2#{what => join_failed}, F). + cets_long:run_safely(Info2#{long_task_name => join}, F). join1(LockKey, Info, LocalPid, RemotePid) -> OtherPids = cets:other_pids(LocalPid), @@ -22,8 +22,7 @@ join1(LockKey, Info, LocalPid, RemotePid) -> {error, already_joined}; false -> Start = erlang:system_time(millisecond), - F = fun() -> join_loop(LockKey, Info, LocalPid, RemotePid, Start) end, - cets_long:run(Info#{task => join}, F) + join_loop(LockKey, Info, LocalPid, RemotePid, Start) end. join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> From 4d88357f97bab2983483adc4d8760323e2981389 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 13 Mar 2023 09:37:14 +0100 Subject: [PATCH 59/64] Add comment about matching result for dialyzer --- src/cets.erl | 1 + src/cets_discovery.erl | 1 + src/cets_mon_cleaner.erl | 1 + 3 files changed, 3 insertions(+) diff --git a/src/cets.erl b/src/cets.erl index 72ce65ee..153fdb07 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -238,6 +238,7 @@ info(Server) -> init({Tab, Opts}) -> process_flag(message_queue_data, off_heap), MonTab = list_to_atom(atom_to_list(Tab) ++ "_mon"), + %% Match result to prevent the Dialyzer warning _ = ets:new(Tab, [ordered_set, named_table, public]), _ = ets:new(MonTab, [public, named_table]), {ok, _} = cets_mon_cleaner:start_link(MonTab, MonTab), diff --git a/src/cets_discovery.erl b/src/cets_discovery.erl index 1fef9d31..8308a44b 100644 --- a/src/cets_discovery.erl +++ b/src/cets_discovery.erl @@ -139,6 +139,7 @@ schedule_check(State) -> State#{timer_ref := TimerRef}. cancel_old_timer(#{timer_ref := OldRef}) when is_reference(OldRef) -> + %% Match result to prevent from Dialyzer warning _ = erlang:cancel_timer(OldRef), flush_all_checks(), ok; diff --git a/src/cets_mon_cleaner.erl b/src/cets_mon_cleaner.erl index 3fb64f16..76fdeef0 100644 --- a/src/cets_mon_cleaner.erl +++ b/src/cets_mon_cleaner.erl @@ -60,6 +60,7 @@ code_change(_OldVsn, State, _Extra) -> -spec schedule_check(state()) -> state(). schedule_check(State = #{interval := Interval, timer_ref := OldRef}) -> + %% Match result to prevent the Dialyzer warning _ = erlang:cancel_timer(OldRef), flush_all_checks(), State#{timer_ref := start_timer(Interval)}. From 2136839aa79966f0360098348476380a74ad0853 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 13 Mar 2023 09:48:46 +0100 Subject: [PATCH 60/64] Add specs for cets_long / cets_join --- src/cets_join.erl | 3 +++ src/cets_long.erl | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/cets_join.erl b/src/cets_join.erl index 44e4497c..17f96e84 100644 --- a/src/cets_join.erl +++ b/src/cets_join.erl @@ -3,9 +3,12 @@ -export([join/4]). -include_lib("kernel/include/logger.hrl"). +-type lock_key() :: term(). + %% Adds a node to a cluster. %% Writes from other nodes would wait for join completion. %% LockKey should be the same on all nodes. +-spec join(lock_key(), cets_long:log_info(), pid(), pid()) -> ok | {error, term()}. join(LockKey, Info, LocalPid, RemotePid) when is_pid(LocalPid), is_pid(RemotePid) -> Info2 = Info#{ local_pid => LocalPid, diff --git a/src/cets_long.erl b/src/cets_long.erl index 22d180c2..fc1e110f 100644 --- a/src/cets_long.erl +++ b/src/cets_long.erl @@ -4,10 +4,19 @@ -include_lib("kernel/include/logger.hrl"). +%% Extra logging information +-type log_info() :: map(). +-type task_result() :: term(). +-type task_fun() :: fun(() -> task_result()). +-type reason() :: {Class :: atom(), Reason :: term(), Stacktrace :: list()}. +-export_type([log_info/0]). + %% Spawn a new process to do some memory-intensive task %% This allows to reduce GC on the parent process %% Wait for function to finish %% Handles errors +%% Returns result from the function or crashes +-spec run_spawn(log_info(), task_fun()) -> task_result(). run_spawn(Info, F) -> Pid = self(), Ref = make_ref(), @@ -20,12 +29,15 @@ run_spawn(Info, F) -> Res end. +-spec run_safely(log_info(), task_fun()) -> task_result() | {error, reason()}. run_safely(Info, Fun) -> run(Info, Fun, true). +-spec run(log_info(), task_fun()) -> task_result(). run(Info, Fun) -> run(Info, Fun, false). +-spec run(log_info(), task_fun(), boolean()) -> task_result() | {error, reason()}. run(Info, Fun, Catch) -> Parent = self(), Start = erlang:system_time(millisecond), From 9efd5afca63ff80085a3e07a62268c372a045524 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 20 Mar 2023 15:52:17 +0100 Subject: [PATCH 61/64] Handle join_with_the_same_pid prevent cets from killing the processes (and losing ETS data) in the event of accidentally calling it with LocalPid same as RemotePid --- src/cets.erl | 15 ++++++++------- src/cets_join.erl | 3 +++ test/cets_SUITE.erl | 11 +++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/cets.erl b/src/cets.erl index 153fdb07..8bf49fe8 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -359,24 +359,25 @@ notify_remote_down_loop(_RemotePid, []) -> %% Merge two lists of pids, create the missing monitors. add_servers(Pids, Servers) -> - lists:sort(add_servers2(Pids, Servers) ++ Servers). + lists:sort(add_servers2(self(), Pids, Servers) ++ Servers). -add_servers2([RemotePid | OtherPids], Servers) when - is_pid(RemotePid), RemotePid =/= self() --> +add_servers2(SelfPid, [SelfPid | OtherPids], Servers) -> + ?LOG_INFO(#{what => join_to_the_same_pid_ignored}), + add_servers2(SelfPid, OtherPids, Servers); +add_servers2(SelfPid, [RemotePid | OtherPids], Servers) when is_pid(RemotePid) -> case has_remote_pid(RemotePid, Servers) of false -> erlang:monitor(process, RemotePid), - [RemotePid | add_servers2(OtherPids, Servers)]; + [RemotePid | add_servers2(SelfPid, OtherPids, Servers)]; true -> ?LOG_INFO(#{ what => already_added, remote_pid => RemotePid, remote_node => node(RemotePid) }), - add_servers2(OtherPids, Servers) + add_servers2(SelfPid, OtherPids, Servers) end; -add_servers2([], _Servers) -> +add_servers2(_SelfPid, [], _Servers) -> []. pids_to_nodes(Pids) -> diff --git a/src/cets_join.erl b/src/cets_join.erl index 17f96e84..a1bb3ab4 100644 --- a/src/cets_join.erl +++ b/src/cets_join.erl @@ -53,6 +53,9 @@ join_loop(LockKey, Info, LocalPid, RemotePid, Start) -> end. join2(_Info, LocalPid, RemotePid) -> + %% Joining is a symmetrical operation here - both servers exchange information between each other. + %% We still use LocalPid/RemotePid in names + %% (they are local and remote pids as passed from the cets_join and from the cets_discovery). LocalOtherPids = cets:other_pids(LocalPid), RemoteOtherPids = cets:other_pids(RemotePid), LocPids = [LocalPid | LocalOtherPids], diff --git a/test/cets_SUITE.erl b/test/cets_SUITE.erl index f6e32492..3f26cdb7 100644 --- a/test/cets_SUITE.erl +++ b/test/cets_SUITE.erl @@ -14,6 +14,7 @@ all() -> inserted_records_could_be_read_back_from_replicated_table, join_works_with_existing_data, join_works_with_existing_data_with_conflicts, + join_with_the_same_pid, test_multinode, node_list_is_correct, test_multinode_auto_discovery, @@ -107,6 +108,16 @@ join_works_with_existing_data_with_conflicts(_Config) -> [{alice, 33}] = ets:lookup(con1tab, alice), [{alice, 32}] = ets:lookup(con2tab, alice). +join_with_the_same_pid(_Config) -> + {ok, Pid} = cets:start(joinsame, #{}), + %% Just insert something into a table to check later the size + cets:insert(joinsame, {1, 1}), + link(Pid), + ok = cets_join:join(joinsame_lock1_con, #{}, Pid, Pid), + Nodes = [node()], + %% The process is still running and no data loss (i.e. size is not zero) + #{nodes := Nodes, size := 1} = cets:info(Pid). + test_multinode(Config) -> Node1 = node(), [Node2, Node3, Node4] = proplists:get_value(nodes, Config), From 665ed3542ca1cf317ff34eb196f356c6bd689cdf Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 20 Mar 2023 16:35:01 +0100 Subject: [PATCH 62/64] Stop monitor process properly Expose mon_pid in cets:info --- src/cets.erl | 16 ++++++++++------ test/cets_SUITE.erl | 11 +++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/cets.erl b/src/cets.erl index 8bf49fe8..e56ff54f 100644 --- a/src/cets.erl +++ b/src/cets.erl @@ -89,6 +89,7 @@ -type state() :: #{ tab := table_name(), mon_tab := atom(), + mon_pid := pid(), other_servers := [pid()], opts := start_opts(), backlog := [backlog_entry()], @@ -110,7 +111,8 @@ table := table_name(), nodes := [node()], size := non_neg_integer(), - memory := non_neg_integer() + memory := non_neg_integer(), + mon_pid := pid() }. -type handle_down_fun() :: fun((#{remote_pid := pid(), table := table_name()}) -> ok). @@ -241,10 +243,11 @@ init({Tab, Opts}) -> %% Match result to prevent the Dialyzer warning _ = ets:new(Tab, [ordered_set, named_table, public]), _ = ets:new(MonTab, [public, named_table]), - {ok, _} = cets_mon_cleaner:start_link(MonTab, MonTab), + {ok, MonPid} = cets_mon_cleaner:start_link(MonTab, MonTab), {ok, #{ tab => Tab, mon_tab => MonTab, + mon_pid => MonPid, other_servers => [], opts => Opts, backlog => [], @@ -303,8 +306,8 @@ handle_info(Msg, State) -> ?LOG_ERROR(#{what => unexpected_info, msg => Msg}), {noreply, State}. -terminate(_Reason, _State) -> - ok. +terminate(_Reason, _State = #{mon_pid := MonPid}) -> + ok = gen_server:stop(MonPid). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -459,12 +462,13 @@ handle_unpause2(Mon, Mons, State) -> {reply, ok, State3}. -spec handle_get_info(state()) -> {reply, info(), state()}. -handle_get_info(State = #{tab := Tab, other_servers := Servers}) -> +handle_get_info(State = #{tab := Tab, other_servers := Servers, mon_pid := MonPid}) -> Info = #{ table => Tab, nodes => lists:usort(pids_to_nodes([self() | Servers])), size => ets:info(Tab, size), - memory => ets:info(Tab, memory) + memory => ets:info(Tab, memory), + mon_pid => MonPid }, {reply, Info, State}. diff --git a/test/cets_SUITE.erl b/test/cets_SUITE.erl index 3f26cdb7..33616f03 100644 --- a/test/cets_SUITE.erl +++ b/test/cets_SUITE.erl @@ -25,6 +25,7 @@ all() -> unpause_twice, write_returns_if_remote_server_crashes, mon_cleaner_works, + mon_cleaner_stops_correctly, sync_using_name_works, insert_many_request ]. @@ -300,6 +301,16 @@ mon_cleaner_works(_Config) -> ok = cets:wait_response(R, 5000), [] = ets:tab2list(c3_mon). +mon_cleaner_stops_correctly(_Config) -> + {ok, Pid} = cets:start(cleaner_stops, #{}), + #{mon_pid := MonPid} = cets:info(Pid), + MonMon = monitor(process, MonPid), + cets:stop(Pid), + receive + {'DOWN', MonMon, process, MonPid, normal} -> ok + after 5000 -> ct:fail(timeout) + end. + sync_using_name_works(_Config) -> {ok, _Pid1} = cets:start(c4, #{}), cets:sync(c4). From b9015bd62c6abb4f1bfc5474b67beb032f8f1377 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 21 Mar 2023 15:29:33 +0100 Subject: [PATCH 63/64] Remove unused cets_long:run/2 --- src/cets_long.erl | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/src/cets_long.erl b/src/cets_long.erl index fc1e110f..2e01e7b2 100644 --- a/src/cets_long.erl +++ b/src/cets_long.erl @@ -1,6 +1,6 @@ %% Helper to log long running operations. -module(cets_long). --export([run_spawn/2, run/2, run_safely/2]). +-export([run_spawn/2, run_safely/2]). -include_lib("kernel/include/logger.hrl"). @@ -31,23 +31,21 @@ run_spawn(Info, F) -> -spec run_safely(log_info(), task_fun()) -> task_result() | {error, reason()}. run_safely(Info, Fun) -> - run(Info, Fun, true). - --spec run(log_info(), task_fun()) -> task_result(). -run(Info, Fun) -> - run(Info, Fun, false). - --spec run(log_info(), task_fun(), boolean()) -> task_result() | {error, reason()}. -run(Info, Fun, Catch) -> Parent = self(), Start = erlang:system_time(millisecond), ?LOG_INFO(Info#{what => long_task_started}), Pid = spawn_mon(Info, Parent, Start), try - case Catch of - true -> just_run_safely(Info#{what => long_task_failed}, Fun); - false -> Fun() - end + Fun() + catch + Class:Reason:Stacktrace -> + ?LOG_ERROR(Info#{ + what => long_task_failed, + class => Class, + reason => Reason, + stacktrace => Stacktrace + }), + {error, {Class, Reason, Stacktrace}} after Diff = diff(Start), ?LOG_INFO(Info#{what => long_task_finished, time_ms => Diff}), @@ -76,12 +74,3 @@ monitor_loop(Mon, Info, Start) -> diff(Start) -> erlang:system_time(millisecond) - Start. - -just_run_safely(Info, Fun) -> - try - Fun() - catch - Class:Reason:Stacktrace -> - ?LOG_ERROR(Info#{class => Class, reason => Reason, stacktrace => Stacktrace}), - {error, {Class, Reason, Stacktrace}} - end. From 3d1e1d63ec3673ca4ecf60b3902727497b889fc1 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 23 Mar 2023 10:22:14 +0100 Subject: [PATCH 64/64] Fix the order of arguments in join in tests --- test/cets_SUITE.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/cets_SUITE.erl b/test/cets_SUITE.erl index 33616f03..7905e051 100644 --- a/test/cets_SUITE.erl +++ b/test/cets_SUITE.erl @@ -127,8 +127,8 @@ test_multinode(Config) -> {ok, Pid2} = start(Node2, Tab), {ok, Pid3} = start(Node3, Tab), {ok, Pid4} = start(Node4, Tab), - ok = join(Node1, Tab, Pid3, Pid1), - ok = join(Node2, Tab, Pid4, Pid2), + ok = join(Node1, Tab, Pid1, Pid3), + ok = join(Node2, Tab, Pid2, Pid4), insert(Node1, Tab, {a}), insert(Node2, Tab, {b}), insert(Node3, Tab, {c}), @@ -167,9 +167,9 @@ node_list_is_correct(Config) -> {ok, Pid2} = start(Node2, Tab), {ok, Pid3} = start(Node3, Tab), {ok, Pid4} = start(Node4, Tab), - ok = join(Node1, Tab, Pid3, Pid1), - ok = join(Node2, Tab, Pid4, Pid2), - ok = join(Node1, Tab, Pid2, Pid1), + ok = join(Node1, Tab, Pid1, Pid3), + ok = join(Node2, Tab, Pid2, Pid4), + ok = join(Node1, Tab, Pid1, Pid2), [Node2, Node3, Node4] = other_nodes(Node1, Tab), [Node1, Node3, Node4] = other_nodes(Node2, Tab), [Node1, Node2, Node4] = other_nodes(Node3, Tab),