diff --git a/big_tests/tests/mam_SUITE.erl b/big_tests/tests/mam_SUITE.erl index 8e8e22c810..87a721af57 100644 --- a/big_tests/tests/mam_SUITE.erl +++ b/big_tests/tests/mam_SUITE.erl @@ -616,7 +616,7 @@ init_per_group(muc_rsm04, Config) -> init_per_group(Group, ConfigIn) -> C = configuration(Group), B = basic_group(Group), - case init_modules(C, B, ConfigIn) of + try init_modules(C, B, ConfigIn) of skip -> {skip, print_configuration_not_supported(C, B)}; Config0 -> @@ -624,7 +624,11 @@ init_per_group(Group, ConfigIn) -> [Group, C, B]), Config1 = do_init_per_group(C, Config0), [{basic_group, B}, {configuration, C} | init_state(C, B, Config1)] - end. + catch Class:Reason:Stacktrace -> + ct:pal("Failed to start configuration=~p basic_group=~p", + [C, B]), + erlang:raise(Class, Reason, Stacktrace) + end. backup_module_opts(Module) -> {{params_backup, Module}, rpc_apply(gen_mod, get_module_opts, [host(), mod_mam_muc])}. @@ -753,7 +757,11 @@ init_modules(rdbms_mnesia_cache, C, Config) when C =:= muc_all; Config; init_modules(BackendType, muc_light, Config) -> Config1 = init_modules_for_muc_light(BackendType, Config), - init_module(host(), mod_mam_rdbms_user, [muc, pm]), + case BackendType of + cassandra -> ok; + elasticsearch -> ok; + _ -> init_module(host(), mod_mam_rdbms_user, [muc, pm]) + end, Config1; init_modules(rdbms, C, Config) -> init_module(host(), mod_mam, addin_mam_options(C, Config)), diff --git a/include/mongoose_mam.hrl b/include/mongoose_mam.hrl new file mode 100644 index 0000000000..b306c31466 --- /dev/null +++ b/include/mongoose_mam.hrl @@ -0,0 +1,2 @@ +-record(db_mapping, {column :: atom(), param :: atom(), format :: atom()}). +-record(lookup_field, {op :: atom(), column :: atom(), param :: atom(), required = false :: boolean(), value_maker :: atom()}). diff --git a/priv/azuresql.sql b/priv/azuresql.sql index a07dea0d07..6d33f4b448 100644 --- a/priv/azuresql.sql +++ b/priv/azuresql.sql @@ -262,7 +262,7 @@ CREATE TABLE [dbo].[privacy_list_data]( [t] [char](1) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL, [value] [varchar](max) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL, [action] [char](1) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL, - [ord] [bigint] NOT NULL, + [ord] [int] NOT NULL, [match_all] [smallint] NOT NULL, [match_iq] [smallint] NOT NULL, [match_message] [smallint] NOT NULL, diff --git a/priv/mssql2012.sql b/priv/mssql2012.sql index 288c92582e..7818ab19a0 100644 --- a/priv/mssql2012.sql +++ b/priv/mssql2012.sql @@ -220,7 +220,7 @@ CREATE TABLE [dbo].[privacy_list_data]( [t] [char](1) NOT NULL, [value] [nvarchar](max) NOT NULL, [action] [char](1) NOT NULL, - [ord] [bigint] NOT NULL, + [ord] [int] NOT NULL, [match_all] [smallint] NOT NULL, [match_iq] [smallint] NOT NULL, [match_message] [smallint] NOT NULL, diff --git a/priv/mysql.sql b/priv/mysql.sql index c9efcc9ae0..172eb11331 100644 --- a/priv/mysql.sql +++ b/priv/mysql.sql @@ -58,7 +58,7 @@ CREATE INDEX i_last_seconds ON last(seconds); CREATE TABLE rosterusers ( username varchar(250) NOT NULL, - jid varchar(250) NOT NULL, + jid varchar(250) NOT NULL, -- must be a parsable jid nick text NOT NULL, subscription character(1) NOT NULL, ask character(1) NOT NULL, @@ -157,7 +157,7 @@ CREATE TABLE privacy_list_data ( t character(1) NOT NULL, value text NOT NULL, action character(1) NOT NULL, - ord bigint NOT NULL, + ord INT NOT NULL, match_all boolean NOT NULL, match_iq boolean NOT NULL, match_message boolean NOT NULL, diff --git a/priv/pg.sql b/priv/pg.sql index 368f91644f..eed593179b 100644 --- a/priv/pg.sql +++ b/priv/pg.sql @@ -143,7 +143,7 @@ CREATE TABLE privacy_list_data ( t character(1) NOT NULL, value text NOT NULL, action character(1) NOT NULL, - ord NUMERIC NOT NULL, + ord INT NOT NULL, match_all boolean NOT NULL, match_iq boolean NOT NULL, match_message boolean NOT NULL, diff --git a/rebar.lock b/rebar.lock index ec362aafe7..a19ec19f0e 100644 --- a/rebar.lock +++ b/rebar.lock @@ -26,7 +26,7 @@ {<<"eini">>,{pkg,<<"eini">>,<<"1.2.7">>},1}, {<<"eodbc">>, {git,"https://github.com/arcusfelis/eodbc.git", - {ref,"612461e4d8e12c0947a67e0bfdeb854010db0b2d"}}, + {ref,"1823d8fe6f5fbe2d8724a9649b75ebd5b8738661"}}, 0}, {<<"eper">>, {git,"http://github.com/basho/eper.git", diff --git a/src/mam/mam_decoder.erl b/src/mam/mam_decoder.erl new file mode 100644 index 0000000000..77b508041d --- /dev/null +++ b/src/mam/mam_decoder.erl @@ -0,0 +1,53 @@ +-module(mam_decoder). +-export([decode_row/2]). +-export([decode_muc_row/2]). +-export([decode_muc_gdpr_row/2]). +-export([decode_retraction_info/2]). + +-type ext_mess_id() :: non_neg_integer() | binary(). +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). +-type db_row() :: {ext_mess_id(), ExtSrcJID :: binary(), ExtData :: binary()}. +-type db_muc_row() :: {ext_mess_id(), Nick :: binary(), ExtData :: binary()}. +-type db_muc_gdpr_row() :: {ext_mess_id(), ExtData :: binary()}. +-type decoded_muc_gdpr_row() :: {ext_mess_id(), exml:element()}. +-type retraction_info() :: #{packet := exml:element(), message_id := mod_mam:message_id()}. + +-spec decode_row(db_row(), env_vars()) -> mod_mam:message_row(). +decode_row({ExtMessID, ExtSrcJID, ExtData}, Env) -> + MessID = mongoose_rdbms:result_to_integer(ExtMessID), + SrcJID = decode_jid(ExtSrcJID, Env), + Packet = decode_packet(ExtData, Env), + {MessID, SrcJID, Packet}. + +-spec decode_muc_row(db_muc_row(), env_vars()) -> mod_mam:message_row(). +decode_muc_row({ExtMessID, Nick, ExtData}, Env = #{archive_jid := RoomJID}) -> + MessID = mongoose_rdbms:result_to_integer(ExtMessID), + SrcJID = jid:replace_resource(RoomJID, Nick), + Packet = decode_packet(ExtData, Env), + {MessID, SrcJID, Packet}. + +-spec decode_muc_gdpr_row(db_muc_gdpr_row(), env_vars()) -> decoded_muc_gdpr_row(). +decode_muc_gdpr_row({ExtMessID, ExtData}, Env) -> + Packet = decode_packet(ExtData, Env), + {ExtMessID, Packet}. + +-spec decode_retraction_info(env_vars(), [] | [{mod_mam:message_id(), binary()}]) -> + skip | retraction_info(). +decode_retraction_info(_Env, []) -> skip; +decode_retraction_info(Env, [{ResMessID, Data}]) -> + Packet = decode_packet(Data, Env), + MessID = mongoose_rdbms:result_to_integer(ResMessID), + #{packet => Packet, message_id => MessID}. + +-spec decode_jid(binary(), env_vars()) -> jid:jid(). +decode_jid(ExtJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> + mam_jid:decode(Codec, ArcJID, ExtJID). + +-spec decode_packet(binary(), env_vars()) -> exml:element(). +decode_packet(ExtBin, Env = #{db_message_codec := Codec}) -> + Bin = unescape_binary(ExtBin, Env), + mam_message:decode(Codec, Bin). + +-spec unescape_binary(binary(), env_vars()) -> binary(). +unescape_binary(Bin, #{host := Host}) -> + mongoose_rdbms:unescape_binary(Host, Bin). diff --git a/src/mam/mam_encoder.erl b/src/mam/mam_encoder.erl new file mode 100644 index 0000000000..c7b58b616f --- /dev/null +++ b/src/mam/mam_encoder.erl @@ -0,0 +1,89 @@ +-module(mam_encoder). +-export([encode_message/3]). +-export([encode_jid/2]). +-export([encode_direction/1]). +-export([encode_packet/2]). +-export([extend_lookup_params/2]). + +-include("mongoose.hrl"). +-include("jlib.hrl"). +-include_lib("exml/include/exml.hrl"). +-include("mongoose_mam.hrl"). + +-type value_type() :: int | maybe_string | direction | bare_jid | jid | jid_resource | xml | search. +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). +-type db_mapping() :: #db_mapping{}. +-type encoded_field_value() :: term(). + +-spec extend_lookup_params(mam_iq:lookup_params(), env_vars()) -> mam_iq:lookup_params(). +extend_lookup_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, + borders := Borders, search_text := SearchText} = Params, Env) -> + Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText), + start_id => make_start_id(Start, Borders), + end_id => make_end_id(End, Borders), + remote_bare_jid => maybe_encode_bare_jid(WithJID, Env), + remote_resource => jid_to_non_empty_resource(WithJID)}. + +-spec encode_message(mod_mam:archive_message_params(), env_vars(), list(db_mapping())) -> + [encoded_field_value()]. +encode_message(Params, Env, Mappings) -> + [encode_value_using_mapping(Params, Env, Mapping) || Mapping <- Mappings]. + +encode_value_using_mapping(Params, Env, #db_mapping{param = Param, format = Format}) -> + Value = maps:get(Param, Params), + encode_value(Format, Value, Env). + +-spec encode_value(value_type(), term(), env_vars()) -> encoded_field_value(). +encode_value(int, Value, _Env) when is_integer(Value) -> + Value; +encode_value(maybe_string, none, _Env) -> + null; +encode_value(maybe_string, Value, _Env) when is_binary(Value) -> + Value; +encode_value(direction, Value, _Env) -> + encode_direction(Value); +encode_value(bare_jid, Value, Env) -> + encode_jid(jid:to_bare(Value), Env); +encode_value(jid, Value, Env) -> + encode_jid(Value, Env); +encode_value(jid_resource, #jid{lresource = Res}, _Env) -> + Res; +encode_value(xml, Value, Env) -> + encode_packet(Value, Env); +encode_value(search, Value, Env) -> + encode_search_body(Value, Env). + +encode_direction(incoming) -> <<"I">>; +encode_direction(outgoing) -> <<"O">>. + +make_start_id(Start, Borders) -> + StartID = maybe_encode_compact_uuid(Start, 0), + mod_mam_utils:apply_start_border(Borders, StartID). + +make_end_id(End, Borders) -> + EndID = maybe_encode_compact_uuid(End, 255), + mod_mam_utils:apply_end_border(Borders, EndID). + +maybe_encode_compact_uuid(undefined, _) -> + undefined; +maybe_encode_compact_uuid(Microseconds, NodeID) -> + mod_mam_utils:encode_compact_uuid(Microseconds, NodeID). + +jid_to_non_empty_resource(undefined) -> undefined; +jid_to_non_empty_resource(#jid{lresource = <<>>}) -> undefined; +jid_to_non_empty_resource(#jid{lresource = Res}) -> Res. + +-spec encode_jid(jid:jid(), env_vars()) -> binary(). +encode_jid(JID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> + mam_jid:encode(Codec, ArcJID, JID). + +maybe_encode_bare_jid(undefined, _Env) -> undefined; +maybe_encode_bare_jid(JID, Env) -> encode_jid(jid:to_bare(JID), Env). + +-spec encode_packet(exml:element(), env_vars()) -> binary(). +encode_packet(Packet, #{db_message_codec := Codec}) -> + mam_message:encode(Codec, Packet). + +-spec encode_search_body(exml:element(), env_vars()) -> binary(). +encode_search_body(Packet, #{has_full_text_search := SearchEnabled}) -> + mod_mam_utils:packet_to_search_body(SearchEnabled, Packet). diff --git a/src/mam/mam_filter.erl b/src/mam/mam_filter.erl new file mode 100644 index 0000000000..4bb1edc3d6 --- /dev/null +++ b/src/mam/mam_filter.erl @@ -0,0 +1,52 @@ +%% Produces filters based on lookup params +-module(mam_filter). +-export([produce_filter/2]). +-include("mongoose_mam.hrl"). + +-type column() :: atom(). + +-type filter_field() :: {like, column(), binary()} + | {le, column(), integer()} + | {ge, column(), integer()} + | {equal, column(), integer() | binary()} + | {less, column(), integer()} + | {greater, column(), integer()}. + +-type filter() :: [filter_field()]. +-type fields() :: [#lookup_field{}]. +-type params() :: map(). + +-export_type([filter_field/0]). +-export_type([filter/0]). + +-define(SEARCH_WORDS_LIMIT, 10). + +-spec produce_filter(params(), fields()) -> list(filter_field()). +produce_filter(Params, Fields) -> + [new_filter(Field, Value) + || Field <- Fields, + Value <- field_to_values(Field, Params)]. + +field_to_values(#lookup_field{param = Param, value_maker = ValueMaker, required = Required} = Field, Params) -> + case maps:find(Param, Params) of + {ok, Value} when Value =/= undefined -> + make_value(ValueMaker, Value); + Other when Required -> + error(#{reason => missing_required_field, field => Field, params => Params, result => Other}); + _ -> + [] + end. + +make_value(search_words, Value) -> search_words(Value); +make_value(undefined, Value) -> [Value]. %% Default value_maker + +new_filter(#lookup_field{op = Op, column = Column}, Value) -> + {Op, Column, Value}. + +%% Constructs a separate LIKE filter for each word. +%% SearchText example is "word1%word2%word3". +%% Order of words does not matter (they can go in any order). +-spec search_words(binary()) -> list(binary()). +search_words(SearchText) -> + Words = binary:split(SearchText, <<"%">>, [global]), + [<<"%", Word/binary, "%">> || Word <- lists:sublist(Words, ?SEARCH_WORDS_LIMIT)]. diff --git a/src/mam/mam_lookup.erl b/src/mam/mam_lookup.erl new file mode 100644 index 0000000000..f25db76fa7 --- /dev/null +++ b/src/mam/mam_lookup.erl @@ -0,0 +1,237 @@ +%% RSM logic lives here +-module(mam_lookup). +-export([lookup/3]). + +-include("mongoose.hrl"). +-include("jlib.hrl"). +-include("mongoose_rsm.hrl"). + +-type filter() :: mam_filter:filter(). +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). +-type params() :: map(). +-type message_id() :: mod_mam:message_id(). +-type maybe_rsm() :: #rsm_in{} | undefined. +-type opt_count_type() :: last_page | by_offset | none. + +%% Public logic +%% We use two fields from Env: +%% - lookup_fn +%% - decode_row_fn +-spec lookup(env_vars(), filter(), params()) -> + {ok, mod_mam:lookup_result()} | {error, item_not_found}. +lookup(Env = #{}, Filter, Params = #{rsm := RSM}) when is_list(Filter) -> + OptParams = Params#{opt_count_type => opt_count_type(RSM)}, + choose_lookup_messages_strategy(Env, Filter, OptParams). + +lookup_query(QueryType, #{lookup_fn := LookupF} = Env, Filters, Order, OffsetLimit) -> + LookupF(QueryType, Env, Filters, Order, OffsetLimit). + +decode_row(Row, #{decode_row_fn := DecodeF} = Env) -> + DecodeF(Row, Env). + +%% Private logic below + +%% There are no optimizations for these queries yet: +%% - #rsm_in{direction = aft, id = ID} +%% - #rsm_in{direction = before, id = ID} +-spec opt_count_type(RSM :: maybe_rsm()) -> opt_count_type(). +opt_count_type(#rsm_in{direction = before, id = undefined}) -> + last_page; %% last page is supported +opt_count_type(#rsm_in{direction = undefined}) -> + by_offset; %% offset +opt_count_type(undefined) -> + by_offset; %% no RSM +opt_count_type(_) -> + none. %% id field is defined in RSM + +%% There are several strategies how to extract messages: +%% - we can use regular query that requires counting; +%% - we can reduce number of queries if we skip counting for small data sets; +%% - sometimes we want not to count at all +%% (for example, our client side counts ones and keep the information) +choose_lookup_messages_strategy(Env, Filter, + Params = #{rsm := RSM, page_size := PageSize}) -> + case Params of + #{is_simple := true} -> + %% Simple query without calculating offset and total count + simple_lookup_messages(Env, RSM, PageSize, Filter); + %% NOTICE: We always prefer opt_count optimization, if possible. + %% Clients don't event know what opt_count is. + #{opt_count_type := last_page} when PageSize > 0 -> + %% Extract messages before calculating offset and total count + %% Useful for small result sets + lookup_last_page(Env, PageSize, Filter); + #{opt_count_type := by_offset} when PageSize > 0 -> + %% Extract messages before calculating offset and total count + %% Useful for small result sets + lookup_by_offset(Env, RSM, PageSize, Filter); + _ -> + %% Calculate offset and total count first before extracting messages + lookup_messages_regular(Env, RSM, PageSize, Filter) + end. + +%% Just extract messages without count and offset information +simple_lookup_messages(Env, RSM, PageSize, Filter) -> + {Filter2, Offset, Order} = rsm_to_filter(RSM, Filter), + Messages = extract_messages(Env, Filter2, Offset, PageSize, Order), + {ok, {undefined, undefined, Messages}}. + +rsm_to_filter(RSM, Filter) -> + case RSM of + %% Get last rows from result set + #rsm_in{direction = aft, id = ID} -> + {after_id(ID, Filter), 0, asc}; + #rsm_in{direction = before, id = undefined} -> + {Filter, 0, desc}; + #rsm_in{direction = before, id = ID} -> + {before_id(ID, Filter), 0, desc}; + #rsm_in{direction = undefined, index = Index} -> + {Filter, Index, asc}; + undefined -> + {Filter, 0, asc} + end. + +%% This function handles case: #rsm_in{direction = before, id = undefined} +%% Assumes assert_rsm_without_id(RSM) +lookup_last_page(Env, PageSize, Filter) -> + Messages = extract_messages(Env, Filter, 0, PageSize, desc), + Selected = length(Messages), + Offset = + case Selected < PageSize of + true -> + 0; %% Result fits on a single page + false -> + FirstID = decoded_row_to_message_id(hd(Messages)), + calc_count(Env, before_id(FirstID, Filter)) + end, + {ok, {Offset + Selected, Offset, Messages}}. + +lookup_by_offset(Env, RSM, PageSize, Filter) -> + assert_rsm_without_id(RSM), + Offset = rsm_to_index(RSM), + Messages = extract_messages(Env, Filter, Offset, PageSize, asc), + Selected = length(Messages), + TotalCount = + case Selected < PageSize of + true -> + Offset + Selected; %% Result fits on a single page + false -> + LastID = decoded_row_to_message_id(lists:last(Messages)), + CountAfterLastID = calc_count(Env, after_id(LastID, Filter)), + Offset + Selected + CountAfterLastID + end, + {ok, {TotalCount, Offset, Messages}}. + +assert_rsm_without_id(undefined) -> ok; +assert_rsm_without_id(#rsm_in{id = undefined}) -> ok. + +rsm_to_index(#rsm_in{direction = undefined, index = Offset}) + when is_integer(Offset) -> Offset; +rsm_to_index(_) -> 0. + +lookup_messages_regular(Env, RSM, PageSize, Filter) -> + TotalCount = calc_count(Env, Filter), + Offset = calc_offset(Env, Filter, PageSize, TotalCount, RSM), + {LookupById, Filter2, Offset2, PageSize2, Order} = + rsm_to_regular_lookup_vars(RSM, Filter, Offset, PageSize), + Messages = extract_messages(Env, Filter2, Offset2, PageSize2, Order), + Result = {TotalCount, Offset, Messages}, + case LookupById of + true -> %% check if we've selected a message with #rsm_in.id + mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); + false -> + {ok, Result} + end. + +rsm_to_regular_lookup_vars(RSM, Filter, Offset, PageSize) -> + case RSM of + #rsm_in{direction = aft, id = ID} when ID =/= undefined -> + %% Set extra flag when selecting PageSize + 1 messages + {true, from_id(ID, Filter), 0, PageSize + 1, asc}; + #rsm_in{direction = before, id = ID} when ID =/= undefined -> + {true, to_id(ID, Filter), 0, PageSize + 1, desc}; + _ -> + {false, Filter, Offset, PageSize, asc} + end. + +decode_rows(MessageRows, Env) -> + [decode_row(Row, Env) || Row <- MessageRows]. + +%% First element of the tuple is decoded message ID +decoded_row_to_message_id(DecodedRow) -> element(1, DecodedRow). + +-spec extract_messages(Env :: env_vars(), + Filter :: filter(), Offset :: non_neg_integer(), Max :: pos_integer(), + Order :: asc | desc) -> [mod_mam:message_row()]. +extract_messages(_Env, _Filter, _Offset, 0 = _Max, _Order) -> + []; +extract_messages(Env, Filter, Offset, Max, Order) -> + {selected, MessageRows} = extract_rows(Env, Filter, Offset, Max, Order), + Rows = maybe_reverse(Order, MessageRows), + decode_rows(Rows, Env). + +maybe_reverse(asc, List) -> List; +maybe_reverse(desc, List) -> lists:reverse(List). + +extract_rows(Env, Filters, Offset, Max, Order) -> + lookup_query(lookup, Env, Filters, Order, {Offset, Max}). + +%% @doc Get the total result set size. +%% SELECT COUNT(*) as count FROM mam_message +-spec calc_count(env_vars(), filter()) -> non_neg_integer(). +calc_count(Env, Filter) -> + Result = lookup_query(count, Env, Filter, unordered, all), + mongoose_rdbms:selected_to_integer(Result). + +%% @doc Calculate a zero-based index of the row with UID in the result test. +%% +%% If the element does not exists, the ID of the next element will +%% be returned instead. +%% @end +%% SELECT COUNT(*) as index FROM mam_message WHERE id <= ? +-spec calc_index(env_vars(), filter(), message_id()) -> non_neg_integer(). +calc_index(Env, Filter, ID) -> + calc_count(Env, to_id(ID, Filter)). + +%% @doc Count of elements in RSet before the passed element. +%% +%% The element with the passed UID can be already deleted. +%% @end +%% SELECT COUNT(*) as count FROM mam_message WHERE id < ? +-spec calc_before(env_vars(), filter(), message_id()) -> non_neg_integer(). +calc_before(Env, Filter, ID) -> + calc_count(Env, before_id(ID, Filter)). + +-spec calc_offset(Env :: env_vars(), + Filter :: filter(), PageSize :: non_neg_integer(), + TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in()) -> non_neg_integer(). +calc_offset(Env, Filter, PageSize, TotalCount, RSM) -> + case RSM of + #rsm_in{direction = undefined, index = Index} when is_integer(Index) -> + Index; + #rsm_in{direction = before, id = undefined} -> + %% Requesting the Last Page in a Result Set + max(0, TotalCount - PageSize); + #rsm_in{direction = before, id = ID} when is_integer(ID) -> + max(0, calc_before(Env, Filter, ID) - PageSize); + #rsm_in{direction = aft, id = ID} when is_integer(ID) -> + calc_index(Env, Filter, ID); + _ -> + 0 + end. + +-spec after_id(message_id(), filter()) -> filter(). +after_id(ID, Filter) -> + [{greater, id, ID}|Filter]. + +-spec before_id(message_id(), filter()) -> filter(). +before_id(ID, Filter) -> + [{less, id, ID}|Filter]. + +-spec from_id(message_id(), filter()) -> filter(). +from_id(ID, Filter) -> + [{ge, id, ID}|Filter]. + +-spec to_id(message_id(), filter()) -> filter(). +to_id(ID, Filter) -> + [{le, id, ID}|Filter]. diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl new file mode 100644 index 0000000000..670a49a98b --- /dev/null +++ b/src/mam/mam_lookup_sql.erl @@ -0,0 +1,145 @@ +%% Makes a SELECT SQL query +-module(mam_lookup_sql). +-export([lookup_query/5]). + +-include("mongoose_logger.hrl"). +-include("mongoose_mam.hrl"). + +-type offset_limit() :: all | {Offset :: non_neg_integer(), Limit :: non_neg_integer()}. +-type sql_part() :: iolist() | binary(). +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). +-type query_type() :: atom(). +-type column() :: atom(). +-type lookup_query_fn() :: fun((QueryType :: atom(), Env :: map(), Filters :: list(), + Order :: atom(), OffsetLimit :: offset_limit()) -> term()). + +-export_type([sql_part/0]). +-export_type([query_type/0]). +-export_type([column/0]). +-export_type([lookup_query_fn/0]). + +%% The ONLY usage of Env is in these functions: +%% The rest of code should treat Env as opaque (i.e. the code just passes Env around). +-spec host(env_vars()) -> jid:lserver(). +host(#{host := Host}) -> Host. + +-spec table(env_vars()) -> atom(). +table(#{table := Table}) -> Table. + +-spec index_hint_sql(env_vars()) -> sql_part(). +index_hint_sql(Env = #{index_hint_fn := F}) -> F(Env). + +-spec columns_sql(env_vars(), query_type()) -> sql_part(). +columns_sql(#{columns_sql_fn := F}, QueryType) -> F(QueryType). + +-spec column_to_id(env_vars(), column()) -> string(). +column_to_id(#{column_to_id_fn := F}, Col) -> F(Col). + + +%% This function uses some fields from Env: +%% - host +%% - table +%% - index_hint_fn +%% - columns_sql_fn +%% - column_to_id_fn +%% +%% Filters are in format {Op, Column, Value} +%% QueryType should be an atom, that we pass into the columns_sql_fn function. +-spec lookup_query(QueryType :: atom(), Env :: map(), Filters :: list(), + Order :: atom(), OffsetLimit :: offset_limit()) -> term(). +lookup_query(QueryType, Env, Filters, Order, OffsetLimit) -> + Table = table(Env), + Host = host(Env), + StmtName = filters_to_statement_name(Env, QueryType, Table, Filters, Order, OffsetLimit), + case mongoose_rdbms:prepared(StmtName) of + false -> + %% Create a new type of a query + SQL = lookup_sql_binary(QueryType, Table, Env, Filters, Order, OffsetLimit), + Columns = filters_to_columns(Filters, OffsetLimit), + mongoose_rdbms:prepare(StmtName, Table, Columns, SQL); + true -> + ok + end, + Args = filters_to_args(Filters, OffsetLimit), + mongoose_rdbms:execute_successfully(Host, StmtName, Args). + +lookup_sql_binary(QueryType, Table, Env, Filters, Order, OffsetLimit) -> + iolist_to_binary(lookup_sql(QueryType, Table, Env, Filters, Order, OffsetLimit)). + +lookup_sql(QueryType, Table, Env, Filters, Order, OffsetLimit) -> + IndexHintSQL = index_hint_sql(Env), + FilterSQL = filters_to_sql(Filters), + OrderSQL = order_to_sql(Order), + {LimitSQL, TopSQL} = limit_sql(OffsetLimit), + ["SELECT ", TopSQL, " ", columns_sql(Env, QueryType), + " FROM ", atom_to_list(Table), " ", + IndexHintSQL, FilterSQL, OrderSQL, LimitSQL]. + +limit_sql(all) -> {"", ""}; +limit_sql({0, _Limit}) -> rdbms_queries:get_db_specific_limits(); +limit_sql({_Offset, _Limit}) -> {rdbms_queries:limit_offset_sql(), ""}. + +filters_to_columns(Filters, OffsetLimit) -> + offset_limit_to_columns(OffsetLimit, [Column || {_Op, Column, _Value} <- Filters]). + +filters_to_args(Filters, OffsetLimit) -> + offset_limit_to_args(OffsetLimit, [Value || {_Op, _Column, Value} <- Filters]). + +offset_limit_to_args(all, Args) -> + Args; +offset_limit_to_args({0, Limit}, Args) -> + rdbms_queries:add_limit_arg(Limit, Args); +offset_limit_to_args({Offset, Limit}, Args) -> + Args ++ rdbms_queries:limit_offset_args(Limit, Offset). + +offset_limit_to_columns(all, Columns) -> + Columns; +offset_limit_to_columns({0, _Limit}, Columns) -> + rdbms_queries:add_limit_arg(limit, Columns); +offset_limit_to_columns({_Offset, _Limit}, Columns) -> + Columns ++ rdbms_queries:limit_offset_args(limit, offset). + +filters_to_statement_name(Env, QueryType, Table, Filters, Order, OffsetLimit) -> + QueryId = query_type_to_id(QueryType), + Ids = [op_to_id(Op) ++ column_to_id(Env, Col) || {Op, Col, _Val} <- Filters], + OrderId = order_type_to_id(Order), + LimitId = offset_limit_to_id(OffsetLimit), + list_to_atom(atom_to_list(Table) ++ "_" ++ QueryId ++ "_" ++ OrderId ++ "_" ++ lists:append(Ids) ++ "_" ++ LimitId). + +query_type_to_id(QueryType) -> atom_to_list(QueryType). + +order_type_to_id(desc) -> "d"; +order_type_to_id(asc) -> "a"; +order_type_to_id(unordered) -> "u". + +order_to_sql(asc) -> " ORDER BY id "; +order_to_sql(desc) -> " ORDER BY id DESC "; +order_to_sql(unordered) -> " ". + +offset_limit_to_id({0, _Limit}) -> "limit"; +offset_limit_to_id({_Offset, _Limit}) -> "offlim"; +offset_limit_to_id(all) -> "all". + +filters_to_sql(Filters) -> + SQLs = [filter_to_sql(Filter) || Filter <- Filters], + case SQLs of + [] -> ""; + Defined -> [" WHERE ", rdbms_queries:join(Defined, " AND ")] + end. + +-spec filter_to_sql(mam_filter:filter_field()) -> sql_part(). +filter_to_sql({Op, Column, _Value}) -> filter_to_sql(atom_to_list(Column), Op). + +op_to_id(equal) -> "eq"; +op_to_id(less) -> "lt"; %% less than +op_to_id(greater) -> "gt"; %% greater than +op_to_id(le) -> "le"; %% less or equal +op_to_id(ge) -> "ge"; %% greater or equal +op_to_id(like) -> "lk". + +filter_to_sql(Column, equal) -> Column ++ " = ?"; +filter_to_sql(Column, less) -> Column ++ " < ?"; +filter_to_sql(Column, greater) -> Column ++ " > ?"; +filter_to_sql(Column, le) -> Column ++ " <= ?"; +filter_to_sql(Column, ge) -> Column ++ " >= ?"; +filter_to_sql(Column, like) -> Column ++ " LIKE ?". diff --git a/src/mam/mod_mam.erl b/src/mam/mod_mam.erl index ddecc35fa7..1d7392630a 100644 --- a/src/mam/mod_mam.erl +++ b/src/mam/mod_mam.erl @@ -118,9 +118,11 @@ -type archive_id() :: non_neg_integer(). -type borders() :: #mam_borders{}. + +-type message_row() :: {message_id(), jid:jid(), exml:element()}. -type lookup_result() :: {TotalCount :: non_neg_integer() | undefined, Offset :: non_neg_integer() | undefined, - MessageRows :: [{message_id(), jid:jid(), exml:element()}]}. + MessageRows :: [message_row()]}. %% Internal types -type iterator_fun() :: fun(() -> {'ok', {_, _}}). @@ -140,7 +142,9 @@ source_jid := jid:jid(), origin_id := binary() | none, direction := atom(), - packet := exml:element()}. + packet := exml:element(), + %% Only in mod_mam_muc_rdbms_arch:retract_message/2 + sender_id => mod_mam:archive_id()}. -export_type([rewriter_fun/0, borders/0, @@ -150,6 +154,7 @@ unix_timestamp/0, archive_id/0, lookup_result/0, + message_row/0, message_id/0, restore_option/0, archive_message_params/0 diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 4d33cdfb6e..d4ba4ba68e 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -1,7 +1,7 @@ %%%------------------------------------------------------------------- %%% @author Uvarov Michael %%% @copyright (C) 2013, Uvarov Michael -%%% @doc A backend for storing messages from MUC rooms using RDBMS. +%%% @doc RDBMS backend for MUC Message Archive Management. %%% @end %%%------------------------------------------------------------------- -module(mod_mam_muc_rdbms_arch). @@ -14,6 +14,8 @@ %% MAM hook handlers -behaviour(ejabberd_gen_mam_archive). +-behaviour(gen_mod). +-behaviour(mongoose_module_metrics). -callback encode(term()) -> binary(). -callback decode(binary()) -> term(). @@ -23,65 +25,58 @@ lookup_messages/3, remove_archive/4]). +-export([get_mam_muc_gdpr_data/2]). + %% Called from mod_mam_rdbms_async_writer -export([prepare_message/2, retract_message/2, prepare_insert/2]). - -%gdpr --export([get_mam_muc_gdpr_data/2]). +-export([extend_params_with_sender_id/2]). %% ---------------------------------------------------------------------- %% Imports -%% UMessID --import(mod_mam_utils, - [encode_compact_uuid/2]). - -%% Other --import(mod_mam_utils, - [apply_start_border/2, - apply_end_border/2]). - --import(mongoose_rdbms, - [escape_integer/1, - use_escaped_string/1, - use_escaped_integer/1]). - -include("mongoose.hrl"). -include("jlib.hrl"). -include_lib("exml/include/exml.hrl"). -include("mongoose_rsm.hrl"). - +-include("mongoose_mam.hrl"). %% ---------------------------------------------------------------------- %% Types --type filter() :: iolist(). --type escaped_message_id() :: mongoose_rdbms:escaped_integer(). --type escaped_room_id() :: mongoose_rdbms:escaped_integer(). --type escaped_jid() :: mongoose_rdbms:escaped_string(). --type unix_timestamp() :: mod_mam:unix_timestamp(). --type packet() :: any(). --type raw_row() :: {binary(), binary(), binary()}. +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). %% ---------------------------------------------------------------------- %% gen_mod callbacks %% Starting and stopping functions for users' archives --spec start(jid:server(), _) -> 'ok'. +-spec start(jid:server(), _) -> ok. start(Host, Opts) -> - prepare_insert(insert_mam_muc_message, 1), - - start_muc(Host, Opts). + start_hooks(Host, Opts), + register_prepared_queries(), + ok. --spec stop(jid:server()) -> 'ok'. +-spec stop(jid:server()) -> ok. stop(Host) -> - stop_muc(Host). + stop_hooks(Host). + +-spec get_mam_muc_gdpr_data(ejabberd_gen_mam_archive:mam_pm_gdpr_data(), jid:jid()) -> + ejabberd_gen_mam_archive:mam_muc_gdpr_data(). +get_mam_muc_gdpr_data(Acc, #jid{luser = User, lserver = Host} = _UserJID) -> + case mod_mam:archive_id(Host, User) of + undefined -> + Acc; + SenderID -> + %% We don't know the real room JID here, use FakeEnv + FakeEnv = env_vars(Host, jid:make(<<>>, <<>>, <<>>)), + {selected, Rows} = extract_gdpr_messages(Host, SenderID), + [mam_decoder:decode_muc_gdpr_row(Row, FakeEnv) || Row <- Rows] ++ Acc + end. %% ---------------------------------------------------------------------- -%% Add hooks for mod_mam_muc +%% Add hooks for mod_mam --spec start_muc(jid:server(), _) -> 'ok'. -start_muc(Host, _Opts) -> +-spec start_hooks(jid:server(), _) -> ok. +start_hooks(Host, _Opts) -> case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of true -> ok; @@ -95,8 +90,8 @@ start_muc(Host, _Opts) -> ok. --spec stop_muc(jid:server()) -> 'ok'. -stop_muc(Host) -> +-spec stop_hooks(jid:server()) -> ok. +stop_hooks(Host) -> case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of true -> ok; @@ -109,541 +104,206 @@ stop_muc(Host) -> ejabberd_hooks:delete(get_mam_muc_gdpr_data, Host, ?MODULE, get_mam_muc_gdpr_data, 50), ok. +%% ---------------------------------------------------------------------- +%% SQL queries + +register_prepared_queries() -> + prepare_insert(insert_mam_muc_message, 1), + mongoose_rdbms:prepare(mam_muc_archive_remove, mam_muc_message, [room_id], + <<"DELETE FROM mam_muc_message " + "WHERE room_id = ?">>), + mongoose_rdbms:prepare(mam_muc_make_tombstone, mam_muc_message, [message, room_id, id], + <<"UPDATE mam_muc_message SET message = ?, search_body = '' " + "WHERE room_id = ? AND id = ?">>), + {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(1), + mongoose_rdbms:prepare(mam_muc_select_messages_to_retract, mam_muc_message, + [room_id, sender_id, origin_id], + <<"SELECT ", LimitMSSQL/binary, + " id, message FROM mam_muc_message" + " WHERE room_id = ? AND sender_id = ? " + " AND origin_id = ?" + " ORDER BY id DESC ", LimitSQL/binary>>), + mongoose_rdbms:prepare(mam_muc_extract_gdpr_messages, mam_muc_message, [sender_id], + <<"SELECT id, message FROM mam_muc_message " + " WHERE sender_id = ? ORDER BY id">>). + +%% ---------------------------------------------------------------------- +%% Declarative logic + +db_mappings() -> + [#db_mapping{column = id, param = message_id, format = int}, + #db_mapping{column = room_id, param = archive_id, format = int}, + #db_mapping{column = sender_id, param = sender_id, format = int}, + #db_mapping{column = nick_name, param = source_jid, format = jid_resource}, + #db_mapping{column = origin_id, param = origin_id, format = maybe_string}, + #db_mapping{column = message, param = packet, format = xml}, + #db_mapping{column = search_body, param = packet, format = search}]. + +lookup_fields() -> + [#lookup_field{op = equal, column = room_id, param = archive_id, required = true}, + #lookup_field{op = ge, column = id, param = start_id}, + #lookup_field{op = le, column = id, param = end_id}, + #lookup_field{op = equal, column = nick_name, param = remote_resource}, + #lookup_field{op = like, column = search_body, param = norm_search_text, value_maker = search_words}]. + +env_vars(Host, ArcJID) -> + %% Please, minimize the usage of the host field. + %% It's only for passing into RDBMS. + #{host => Host, + archive_jid => ArcJID, + table => mam_muc_message, + index_hint_fn => fun index_hint_sql/1, + columns_sql_fn => fun columns_sql/1, + column_to_id_fn => fun column_to_id/1, + lookup_fn => fun lookup_query/5, + decode_row_fn => fun row_to_uniform_format/2, + has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam_muc, Host), + has_full_text_search => mod_mam_utils:has_full_text_search(mod_mam_muc, Host), + db_jid_codec => db_jid_codec(Host, ?MODULE), + db_message_codec => db_message_codec(Host, ?MODULE)}. + +row_to_uniform_format(Row, Env) -> + mam_decoder:decode_muc_row(Row, Env). + +-spec index_hint_sql(env_vars()) -> string(). +index_hint_sql(_) -> "". + +columns_sql(lookup) -> "id, nick_name, message"; +columns_sql(count) -> "COUNT(*)". + +column_to_id(id) -> "i"; +column_to_id(room_id) -> "u"; +column_to_id(nick_name) -> "n"; +column_to_id(search_body) -> "s". + +column_names(Mappings) -> + [Column || #db_mapping{column = Column} <- Mappings]. + +%% ---------------------------------------------------------------------- +%% Options + +-spec db_jid_codec(jid:server(), module()) -> module(). +db_jid_codec(Host, Module) -> + gen_mod:get_module_opt(Host, Module, db_jid_format, mam_jid_rfc). + +-spec db_message_codec(jid:server(), module()) -> module(). +db_message_codec(Host, Module) -> + gen_mod:get_module_opt(Host, Module, db_message_format, mam_message_compressed_eterm). + +-spec get_retract_id(exml:element(), env_vars()) -> none | binary(). +get_retract_id(Packet, #{has_message_retraction := Enabled}) -> + mod_mam_utils:get_retract_id(Enabled, Packet). %% ---------------------------------------------------------------------- %% Internal functions and callbacks --spec archive_size(integer(), jid:server(), integer(), jid:jid()) - -> integer(). -archive_size(Size, Host, RoomID, _RoomJID) when is_integer(Size) -> - {selected, [{BSize}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) " - "FROM mam_muc_message ", - "WHERE room_id = ", use_escaped_integer(escape_room_id(RoomID))]), - mongoose_rdbms:result_to_integer(BSize). +-spec archive_size(Size :: integer(), Host :: jid:server(), + ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). +archive_size(Size, Host, ArcID, ArcJID) when is_integer(Size) -> + Filter = [{equal, room_id, ArcID}], + Env = env_vars(Host, ArcJID), + Result = lookup_query(count, Env, Filter, unordered, all), + mongoose_rdbms:selected_to_integer(Result). + +extend_params_with_sender_id(Host, Params = #{remote_jid := SenderJID}) -> + BareSenderJID = jid:to_bare(SenderJID), + SenderID = mod_mam:archive_id_int(Host, BareSenderJID), + Params#{sender_id => SenderID}. -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok. -archive_message(_Result, Host, Params = #{direction := incoming}) -> +archive_message(_Result, Host, Params0 = #{local_jid := ArcJID}) -> try - Row = prepare_message(Host, Params), - {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_muc_message, Row), - retract_message(Host, Params), + Params = extend_params_with_sender_id(Host, Params0), + Env = env_vars(Host, ArcJID), + do_archive_message(Host, Params, Env), + retract_message(Host, Params, Env), ok - catch _Type:Reason:StackTrace -> - ?LOG_ERROR(#{what => archive_message_failed, - host => Host, mam_params => Params, - reason => Reason, stacktrace => StackTrace}), - {error, Reason} + catch error:Reason:StackTrace -> + ?LOG_ERROR(#{what => archive_message_failed, + host => Host, mam_params => Params0, + reason => Reason, stacktrace => StackTrace}), + erlang:raise(error, Reason, StackTrace) end. -retract_message(Host, #{archive_id := RoomID, - remote_jid := SenderJID, - packet := Packet}) -> - case mod_mam_utils:get_retract_id(mod_mam_muc, Host, Packet) of +do_archive_message(Host, Params, Env) -> + Row = mam_encoder:encode_message(Params, Env, db_mappings()), + {updated, 1} = mongoose_rdbms:execute_successfully(Host, insert_mam_muc_message, Row). + +%% Retraction logic +%% Called after inserting a new message +-spec retract_message(jid:server(), mod_mam:archive_message_params()) -> ok. +retract_message(Host, #{local_jid := ArcJID} = Params) -> + Env = env_vars(Host, ArcJID), + retract_message(Host, Params, Env). + +-spec retract_message(jid:server(), mod_mam:archive_message_params(), env_vars()) -> ok. +retract_message(Host, #{archive_id := ArcID, sender_id := SenderID, + packet := Packet}, Env) -> + case get_retract_id(Packet, Env) of none -> ok; - OriginIDToRetract -> retract_message(Host, RoomID, SenderJID, OriginIDToRetract) + OriginIDToRetract -> + Info = get_retraction_info(Host, ArcID, SenderID, OriginIDToRetract, Env), + make_tombstone(Host, ArcID, OriginIDToRetract, Info, Env) end. -retract_message(Host, RoomID, SenderJID, OriginID) -> - SRoomID = use_escaped_integer(escape_room_id(RoomID)), - SenderID = mod_mam:archive_id_int(Host, jid:to_bare(SenderJID)), - SSenderID = use_escaped_integer(mongoose_rdbms:escape_integer(SenderID)), - SOriginID = use_escaped_string(mongoose_rdbms:escape_string(OriginID)), - Query = query_for_messages_to_retract(SRoomID, SSenderID, SOriginID), - {selected, Rows} = mod_mam_utils:success_sql_query(Host, Query), - make_tombstone(Host, SRoomID, OriginID, Rows), - ok. +get_retraction_info(Host, ArcID, SenderID, OriginID, Env) -> + {selected, Rows} = + execute_select_messages_to_retract(Host, ArcID, SenderID, OriginID), + mam_decoder:decode_retraction_info(Env, Rows). -make_tombstone(_Host, SRoomID, OriginID, []) -> +make_tombstone(_Host, ArcID, OriginID, skip, _Env) -> ?LOG_INFO(#{what => make_tombstone_failed, text => <<"Message to retract was not found by origin id">>, - room_id => SRoomID, origin_id => OriginID}); -make_tombstone(Host, SRoomID, OriginID, [{ResMessID, ResData}]) -> - Data = mongoose_rdbms:unescape_binary(Host, ResData), - Packet = stored_binary_to_packet(Host, Data), - MessID = mongoose_rdbms:result_to_integer(ResMessID), + user_id => ArcID, origin_id => OriginID}); +make_tombstone(Host, ArcID, OriginID, #{packet := Packet, message_id := MessID}, Env) -> Tombstone = mod_mam_utils:tombstone(Packet, OriginID), - TombstoneData = packet_to_stored_binary(Host, Tombstone), - STombstoneData = mongoose_rdbms:use_escaped_binary( - mongoose_rdbms:escape_binary(Host, TombstoneData)), - BMessID = use_escaped_integer(escape_message_id(MessID)), - UpdateQuery = query_to_make_tombstone(STombstoneData, SRoomID, BMessID), - {updated, 1} = mod_mam_utils:success_sql_query(Host, UpdateQuery). - -query_for_messages_to_retract(SRoomID, SSenderID, SOriginID) -> - {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits(1), - ["SELECT ", LimitMSSQL, " id, message FROM mam_muc_message" - " WHERE room_id = ", SRoomID, " AND sender_id = ", SSenderID, " AND origin_id = ", SOriginID, - " ORDER BY id DESC ", LimitSQL]. - -query_to_make_tombstone(STombstoneData, SRoomID, BMessID) -> - ["UPDATE mam_muc_message SET message = ", STombstoneData, ", search_body = ''" - " WHERE room_id = ", SRoomID, " AND id = '", BMessID, "'"]. - --spec prepare_message(Host :: jid:server(), Params :: mod_mam:archive_message_params()) -> - [binary() | integer()]. -prepare_message(Host, #{message_id := MessID, - archive_id := RoomID, - remote_jid := SenderJID, - source_jid := #jid{lresource = FromNick}, - origin_id := OriginID, - packet := Packet}) -> - BareSenderJID = jid:to_bare(SenderJID), - Data = packet_to_stored_binary(Host, Packet), - TextBody = mod_mam_utils:packet_to_search_body(mod_mam_muc, Host, Packet), - SenderID = mod_mam:archive_id_int(Host, BareSenderJID), - SOriginID = case OriginID of - none -> null; - _ -> OriginID - end, - [MessID, RoomID, SenderID, FromNick, SOriginID, Data, TextBody]. + TombstoneData = mam_encoder:encode_packet(Tombstone, Env), + execute_make_tombstone(Host, TombstoneData, ArcID, MessID). + +execute_select_messages_to_retract(Host, ArcID, SenderID, OriginID) -> + mongoose_rdbms:execute_successfully(Host, mam_muc_select_messages_to_retract, + [ArcID, SenderID, OriginID]). + +execute_make_tombstone(Host, TombstoneData, ArcID, MessID) -> + mongoose_rdbms:execute_successfully(Host, mam_muc_make_tombstone, + [TombstoneData, ArcID, MessID]). + +%% Insert logic +-spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). +prepare_message(Host, Params = #{local_jid := ArcJID}) -> + Env = env_vars(Host, ArcJID), + mam_encoder:encode_message(Params, Env, db_mappings()). -spec prepare_insert(Name :: atom(), NumRows :: pos_integer()) -> ok. prepare_insert(Name, NumRows) -> Table = mam_muc_message, - Fields = [id, room_id, sender_id, nick_name, origin_id, message, search_body], + Fields = column_names(db_mappings()), {Query, Fields2} = rdbms_queries:create_bulk_insert_query(Table, Fields, NumRows), mongoose_rdbms:prepare(Name, Table, Fields2, Query), ok. - -lookup_messages({error, _Reason}=Result, _Host, _Params) -> - Result; -lookup_messages(_Result, Host, - #{archive_id := UserID, owner_jid := UserJID, rsm := RSM, - borders := Borders, start_ts := Start, end_ts := End, now := Now, - with_jid := WithJID, search_text := SearchText, page_size := PageSize, - is_simple := IsSimple}) -> - try - lookup_messages(Host, - UserID, UserJID, RSM, Borders, - Start, End, Now, WithJID, - mod_mam_utils:normalize_search_text(SearchText), - PageSize, IsSimple) - catch _Type:Reason:S -> - {error, {Reason, {stacktrace, S}}} - end. - --spec lookup_messages(Host :: jid:server(), - ArchiveID :: mod_mam:archive_id(), - ArchiveJID :: jid:jid(), - RSM :: jlib:rsm_in() | undefined, - Borders :: mod_mam:borders() | undefined, - Start :: mod_mam:unix_timestamp() | undefined, - End :: mod_mam:unix_timestamp() | undefined, - Now :: mod_mam:unix_timestamp(), - WithJID :: jid:jid() | undefined, - SearchText :: binary() | undefined, - PageSize :: integer(), - IsSimple :: boolean() | opt_count) -> - {ok, mod_mam:lookup_result()}. -lookup_messages(Host, RoomID, RoomJID = #jid{}, - #rsm_in{direction = aft, id = ID}, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, true) -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, after_id(ID, Filter), 0, PageSize, false), - {ok, {undefined, undefined, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - #rsm_in{direction = before, id = ID}, - Borders, Start, End, _Now, WithJID, SearchText, - PageSize, true) -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, before_id(ID, Filter), 0, PageSize, true), - {ok, {undefined, undefined, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - #rsm_in{direction = undefined, index = Offset}, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, true) -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), - {ok, {undefined, undefined, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - undefined, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, true) -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, Filter, 0, PageSize, false), - {ok, {undefined, undefined, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; -%% Cannot be optimized: -%% - #rsm_in{direction = aft, id = ID} -%% - #rsm_in{direction = before, id = ID} -lookup_messages(Host, RoomID, RoomJID = #jid{}, - #rsm_in{direction = before, id = undefined}, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, opt_count) -> - %% Last page - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, Filter, 0, PageSize, true), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {MessageRowsCount, 0, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; - false -> - FirstID = row_to_message_id(hd(MessageRows)), - Offset = calc_count(Host, before_id(FirstID, Filter)), - {ok, {Offset + MessageRowsCount, Offset, - rows_to_uniform_format(MessageRows, Host, RoomJID)}} - end; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - #rsm_in{direction = undefined, index = Offset}, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, opt_count) -> - %% By offset - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {Offset + MessageRowsCount, Offset, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; - false -> - LastID = row_to_message_id(lists:last(MessageRows)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), - {ok, {Offset + MessageRowsCount + CountAfterLastID, Offset, - rows_to_uniform_format(MessageRows, Host, RoomJID)}} - end; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - undefined, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, opt_count) -> - %% First page - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, Filter, 0, PageSize, false), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {MessageRowsCount, 0, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; - false -> - LastID = row_to_message_id(lists:last(MessageRows)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), - {ok, {MessageRowsCount + CountAfterLastID, 0, - rows_to_uniform_format(MessageRows, Host, RoomJID)}} - end; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - RSM = #rsm_in{direction = aft, id = ID}, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, _) when ID =/= undefined -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - TotalCount = calc_count(Host, Filter), - Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, from_id(ID, Filter), 0, PageSize + 1, false), - Result = {TotalCount, Offset, rows_to_uniform_format(MessageRows, Host, RoomJID)}, - mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); -lookup_messages(Host, RoomID, RoomJID = #jid{}, - RSM = #rsm_in{direction = before, id = ID}, - Borders, Start, End, _Now, WithJID, SearchText, - PageSize, _) when ID =/= undefined -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - TotalCount = calc_count(Host, Filter), - Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, to_id(ID, Filter), 0, PageSize + 1, true), - Result = {TotalCount, Offset, rows_to_uniform_format(MessageRows, Host, RoomJID)}, - mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); -lookup_messages(Host, RoomID, RoomJID = #jid{}, - RSM, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, _) -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - TotalCount = calc_count(Host, Filter), - Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), - {ok, {TotalCount, Offset, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}. - --spec get_mam_muc_gdpr_data(ejabberd_gen_mam_archive:mam_muc_gdpr_data(), jid:jid()) -> - ejabberd_gen_mam_archive:mam_muc_gdpr_data(). -get_mam_muc_gdpr_data(Acc, #jid{ user = User, server = Host }) -> - case mod_mam:archive_id(Host, User) of - undefined -> Acc; - ArchiveID -> - {selected, Rows} = extract_gdpr_messages(Host, ArchiveID), - [{BMessID, gdpr_decode_packet(Host, SDataRaw)} || {BMessID, SDataRaw} <- Rows] ++ Acc - end. - --spec after_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). -after_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id > ", use_escaped_integer(SID)]. - --spec before_id(ID :: escaped_message_id() | undefined, - Filter :: filter()) -> filter(). -before_id(undefined, Filter) -> - Filter; -before_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id < ", use_escaped_integer(SID)]. - --spec from_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). -from_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id >= ", use_escaped_integer(SID)]. - --spec to_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). -to_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id <= ", use_escaped_integer(SID)]. - - --spec rows_to_uniform_format([raw_row()], jid:server(), jid:jid()) -> - [mod_mam_muc:row()]. -rows_to_uniform_format(MessageRows, Host, RoomJID) -> - [do_row_to_uniform_format(Host, Row, RoomJID) || Row <- MessageRows]. - - --spec do_row_to_uniform_format(jid:server(), raw_row(), jid:jid()) -> - mod_mam_muc:row(). -do_row_to_uniform_format(Host, {BMessID, BNick, SDataRaw}, RoomJID) -> - MessID = mongoose_rdbms:result_to_integer(BMessID), - SrcJID = jid:replace_resource(RoomJID, BNick), - Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Packet = stored_binary_to_packet(Host, Data), - {MessID, SrcJID, Packet}. - - --spec row_to_message_id({binary(), _, _}) -> integer(). -row_to_message_id({BMessID, _, _}) -> - mongoose_rdbms:result_to_integer(BMessID). - - --spec remove_archive(map(), jid:server(), mod_mam:archive_id(), jid:jid()) -> map(). -remove_archive(Acc, Host, RoomID, _RoomJID) -> - {updated, _} = - mod_mam_utils:success_sql_query( - Host, - ["DELETE FROM mam_muc_message " - "WHERE room_id = ", use_escaped_integer(escape_room_id(RoomID))]), +%% Removal logic +-spec remove_archive(Acc :: mongoose_acc:t(), Host :: jid:server(), + ArcID :: mod_mam:archive_id(), + ArcJID :: jid:jid()) -> mongoose_acc:t(). +remove_archive(Acc, Host, ArcID, _ArcJID) -> + mongoose_rdbms:execute_successfully(Host, mam_muc_archive_remove, [ArcID]), Acc. -%% @doc Columns are `["id", "nick_name", "message"]'. --spec extract_messages(Host :: jid:server(), - Filter :: filter(), IOffset :: non_neg_integer(), IMax :: pos_integer(), - ReverseLimit :: boolean()) -> [raw_row()]. -extract_messages(_Host, _Filter, _IOffset, 0, _) -> - []; -extract_messages(Host, Filter, IOffset, IMax, false) -> - {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, " ORDER BY id "), - ?LOG_DEBUG(#{what => mam_extract_messages, - text => <<"extract_messages query returns...">>, - mam_filter => Filter, offset => IOffset, max => IMax, - host => Host, message_rows => MessageRows}), - MessageRows; -extract_messages(Host, Filter, IOffset, IMax, true) -> - {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, " ORDER BY id DESC "), - ?LOG_DEBUG(#{what => mam_extract_messages, - text => <<"extract_messages query returns...">>, - mam_filter => Filter, offset => IOffset, max => IMax, - host => Host, message_rows => MessageRows}), - lists:reverse(MessageRows). - -do_extract_messages(Host, Filter, 0, IMax, Order) -> - {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits(IMax), - mod_mam_utils:success_sql_query( - Host, - ["SELECT ", LimitMSSQL, " id, nick_name, message " - "FROM mam_muc_message ", - Filter, - Order, - " ", LimitSQL]); -do_extract_messages(Host, Filter, IOffset, IMax, Order) -> - {LimitSQL, _LimitMSSQL} = rdbms_queries:get_db_specific_limits(IMax), - Offset = rdbms_queries:get_db_specific_offset(IOffset, IMax), - mod_mam_utils:success_sql_query( - Host, - ["SELECT id, nick_name, message " - "FROM mam_muc_message ", - Filter, Order, LimitSQL, Offset]). - -extract_gdpr_messages(Host, ArchiveID) -> - Filter = ["WHERE sender_id = ", use_escaped_integer(escape_integer(ArchiveID))], - mod_mam_utils:success_sql_query( - Host, - ["SELECT id, message " - "FROM mam_muc_message ", - Filter, " ORDER BY id"]). - -%% @doc Zero-based index of the row with UMessID in the result test. -%% If the element does not exists, the MessID of the next element will -%% be returned instead. -%% -%% ``` -%% "SELECT COUNT(*) as "index" FROM mam_muc_message WHERE id <= '", UMessID -%% ''' --spec calc_index(Host :: jid:server(), - Filter :: iodata(), SUMessID :: escaped_message_id()) -> non_neg_integer(). -calc_index(Host, Filter, SUMessID) -> - {selected, [{BIndex}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) " - "FROM mam_muc_message ", - Filter, " AND id <= ", use_escaped_integer(SUMessID)]), - mongoose_rdbms:result_to_integer(BIndex). - - -%% @doc Count of elements in RSet before the passed element. -%% The element with the passed UMessID can be already deleted. -%% @end -%% "SELECT COUNT(*) as "count" FROM mam_muc_message WHERE id < '", UMessID --spec calc_before(Host :: jid:server(), - Filter :: iodata(), SUMessID :: escaped_message_id()) -> non_neg_integer(). -calc_before(Host, Filter, SUMessID) -> - {selected, [{BIndex}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) " - "FROM mam_muc_message ", - Filter, " AND id < ", use_escaped_integer(SUMessID)]), - mongoose_rdbms:result_to_integer(BIndex). - - -%% @doc Get the total result set size. -%% "SELECT COUNT(*) as "count" FROM mam_muc_message WHERE " --spec calc_count(Host :: jid:server(), - Filter :: filter()) -> non_neg_integer(). -calc_count(Host, Filter) -> - {selected, [{BCount}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) ", - "FROM mam_muc_message ", Filter]), - mongoose_rdbms:result_to_integer(BCount). - - -%% @doc prepare_filter/5 --spec prepare_filter(RoomID :: mod_mam:archive_id(), Borders :: mod_mam:borders() | undefined, - Start :: unix_timestamp() | undefined, End :: unix_timestamp() | undefined, - WithJID :: jid:jid() | undefined, SearchText :: binary() | undefined) -> filter(). -prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText) -> - SWithNick = maybe_jid_to_escaped_resource(WithJID), - StartID = maybe_encode_compact_uuid(Start, 0), - EndID = maybe_encode_compact_uuid(End, 255), - StartID2 = apply_start_border(Borders, StartID), - EndID2 = apply_end_border(Borders, EndID), - make_filter(RoomID, StartID2, EndID2, SWithNick, SearchText). - - --spec make_filter(RoomID :: non_neg_integer(), - StartID :: mod_mam:message_id() | undefined, - EndID :: mod_mam:message_id() | undefined, - SWithNick :: escaped_jid() | undefined, - SearchText :: binary() | undefined) -> filter(). -make_filter(RoomID, StartID, EndID, SWithNick, SearchText) -> - ["WHERE room_id=", use_escaped_integer(escape_room_id(RoomID)), - case StartID of - undefined -> ""; - _ -> [" AND id >= ", use_escaped_integer(escape_integer(StartID))] - end, - case EndID of - undefined -> ""; - _ -> [" AND id <= ", use_escaped_integer(escape_integer(EndID))] - end, - case SWithNick of - undefined -> ""; - _ -> [" AND nick_name = ", use_escaped_string(SWithNick)] - end, - case SearchText of - undefined -> ""; - _ -> prepare_search_filters(SearchText) - end - ]. - -%% Constructs a separate LIKE filter for each word. -%% SearchText example is "word1%word2%word3". -prepare_search_filters(SearchText) -> - Words = binary:split(SearchText, <<"%">>, [global]), - [prepare_search_filter(Word) || Word <- Words]. - --spec prepare_search_filter(binary()) -> filter(). -prepare_search_filter(Word) -> - [" AND search_body like ", - %% Search for "%Word%" - mongoose_rdbms:use_escaped_like(mongoose_rdbms:escape_like(Word))]. - -%% @doc #rsm_in{ -%% max = non_neg_integer() | undefined, -%% direction = before | aft | undefined, -%% id = binary() | undefined, -%% index = non_neg_integer() | undefined} --spec calc_offset(Host :: jid:server(), - Filter :: filter(), PageSize :: non_neg_integer(), - TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in() | undefined) - -> non_neg_integer(). -calc_offset(_LS, _F, _PS, _TC, #rsm_in{direction = undefined, index = Index}) - when is_integer(Index) -> - Index; -%% Requesting the Last Page in a Result Set -calc_offset(_LS, _F, PS, TC, #rsm_in{direction = before, id = undefined}) -> - max(0, TC - PS); -calc_offset(Host, F, PS, _TC, #rsm_in{direction = before, id = MessID}) - when is_integer(MessID) -> - SMessID = escape_message_id(MessID), - max(0, calc_before(Host, F, SMessID) - PS); -calc_offset(Host, F, _PS, _TC, #rsm_in{direction = aft, id = MessID}) - when is_integer(MessID) -> - SMessID = escape_message_id(MessID), - calc_index(Host, F, SMessID); -calc_offset(_LS, _F, _PS, _TC, _RSM) -> - 0. - - --spec escape_message_id(mod_mam:message_id()) -> escaped_message_id(). -escape_message_id(MessID) when is_integer(MessID) -> - escape_integer(MessID). - - --spec escape_room_id(mod_mam:archive_id()) -> escaped_room_id(). -escape_room_id(RoomID) when is_integer(RoomID) -> - escape_integer(RoomID). - - --spec maybe_jid_to_escaped_resource('undefined' | jid:jid()) - -> 'undefined' | mongoose_rdbms:escaped_string(). -maybe_jid_to_escaped_resource(undefined) -> - undefined; -maybe_jid_to_escaped_resource(#jid{lresource = <<>>}) -> - undefined; -maybe_jid_to_escaped_resource(#jid{lresource = WithLResource}) -> - mongoose_rdbms:escape_string(WithLResource). - - --spec maybe_encode_compact_uuid('undefined' | integer(), 0 | 255) - -> 'undefined' | integer(). -maybe_encode_compact_uuid(undefined, _) -> - undefined; -maybe_encode_compact_uuid(Microseconds, NodeID) -> - encode_compact_uuid(Microseconds, NodeID). - - -%% ---------------------------------------------------------------------- -%% Optimizations - -packet_to_stored_binary(Host, Packet) -> - Module = db_message_codec(Host), - mam_message:encode(Module, Packet). - -stored_binary_to_packet(Host, Bin) -> - Module = db_message_codec(Host), - mam_message:decode(Module, Bin). - --spec db_message_codec(Host :: jid:server()) -> module(). -db_message_codec(Host) -> - gen_mod:get_module_opt(Host, ?MODULE, db_message_format, mam_message_compressed_eterm). +%% GDPR logic +extract_gdpr_messages(Host, SenderID) -> + mongoose_rdbms:execute_successfully(Host, mam_muc_extract_gdpr_messages, [SenderID]). - -gdpr_decode_packet(Host, SDataRaw) -> - Codec = mod_mam_meta:get_mam_module_opt(Host, ?MODULE, db_message_format, - mam_message_compressed_eterm), - Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Message = mam_message:decode(Codec, Data), - exml:to_binary(Message). +%% Lookup logic +-spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> + {ok, mod_mam:lookup_result()}. +lookup_messages({error, _Reason} = Result, _Host, _Params) -> + Result; +lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> + Env = env_vars(Host, ArcJID), + ExdParams = mam_encoder:extend_lookup_params(Params, Env), + Filter = mam_filter:produce_filter(ExdParams, lookup_fields()), + mam_lookup:lookup(Env, Filter, ExdParams). + +lookup_query(QueryType, Env, Filters, Order, OffsetLimit) -> + mam_lookup_sql:lookup_query(QueryType, Env, Filters, Order, OffsetLimit). diff --git a/src/mam/mod_mam_muc_rdbms_async_pool_writer.erl b/src/mam/mod_mam_muc_rdbms_async_pool_writer.erl index 67ac1d3e92..08b6f4efde 100644 --- a/src/mam/mod_mam_muc_rdbms_async_pool_writer.erl +++ b/src/mam/mod_mam_muc_rdbms_async_pool_writer.erl @@ -153,7 +153,8 @@ stop_worker(Proc) -> -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok | {error, timeout}. -archive_message(_Result, Host, Params = #{archive_id := RoomID}) -> +archive_message(_Result, Host, Params0 = #{archive_id := RoomID}) -> + Params = mod_mam_muc_rdbms_arch:extend_params_with_sender_id(Host, Params0), Worker = select_worker(Host, RoomID), WorkerPid = whereis(Worker), %% Send synchronously if queue length is too long. diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 5a08a96401..496727a3d0 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -33,71 +33,66 @@ %% ---------------------------------------------------------------------- %% Imports -%% UID --import(mod_mam_utils, - [encode_compact_uuid/2]). - -%% Other --import(mod_mam_utils, - [apply_start_border/2, - apply_end_border/2]). - --import(mongoose_rdbms, - [escape_string/1, - escape_integer/1, - use_escaped_string/1, - use_escaped_integer/1]). - -include("mongoose.hrl"). -include("jlib.hrl"). -include_lib("exml/include/exml.hrl"). -include("mongoose_rsm.hrl"). +-include("mongoose_mam.hrl"). %% ---------------------------------------------------------------------- %% Types --type filter() :: mongoose_rdbms:sql_query_part(). --type escaped_message_id() :: mongoose_rdbms:escape_string(). --type escaped_jid() :: mongoose_rdbms:escaped_string(). --type escaped_resource() :: mongoose_rdbms:escaped_string(). - +-type env_vars() :: #{ + host := jid:lserver(), + archive_jid := jid:jid(), + table := atom(), + index_hint_fn := fun((env_vars()) -> mam_lookup_sql:sql_part()), + columns_sql_fn := fun((mam_lookup_sql:query_type()) -> mam_lookup_sql:sql_part()), + column_to_id_fn := fun((mam_lookup_sql:column()) -> string()), + lookup_fn := mam_lookup_sql:lookup_query_fn(), + decode_row_fn := fun((Row :: tuple(), env_vars()) -> Decoded :: tuple()), + has_message_retraction := boolean(), + has_full_text_search := boolean(), + db_jid_codec := module(), + db_message_codec := module() + }. + +-export_type([env_vars/0]). %% ---------------------------------------------------------------------- %% gen_mod callbacks %% Starting and stopping functions for users' archives --spec start(jid:server(), _) -> 'ok'. +-spec start(jid:server(), _) -> ok. start(Host, Opts) -> - start_pm(Host, Opts), - - prepare_insert(insert_mam_message, 1), - mongoose_rdbms:prepare(mam_archive_size, mam_message, [user_id], - [<<"SELECT COUNT(*) FROM mam_message ">>, - index_hint_sql(Host), - <<"WHERE user_id = ?">>]), + start_hooks(Host, Opts), + register_prepared_queries(), ok. - -spec stop(jid:server()) -> ok. stop(Host) -> - stop_pm(Host). + stop_hooks(Host). -spec get_mam_pm_gdpr_data(ejabberd_gen_mam_archive:mam_pm_gdpr_data(), jid:jid()) -> ejabberd_gen_mam_archive:mam_pm_gdpr_data(). -get_mam_pm_gdpr_data(Acc, #jid{ user = User, server = Server, lserver = LServer } = UserJid) -> - case mod_mam:archive_id(Server, User) of - undefined -> []; - ArchiveID -> - {selected, Rows} = extract_gdpr_messages(LServer, ArchiveID), - [{BMessID, gdpr_decode_jid(LServer, UserJid, FromJID), - gdpr_decode_packet(LServer, SDataRaw)} || {BMessID, FromJID, SDataRaw} <- Rows] ++ Acc +get_mam_pm_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> + case mod_mam:archive_id(Host, User) of + undefined -> + Acc; + ArcID -> + Env = env_vars(Host, ArcJID), + {selected, Rows} = extract_gdpr_messages(Env, ArcID), + [uniform_to_gdpr(row_to_uniform_format(Row, Env)) || Row <- Rows] ++ Acc end. +uniform_to_gdpr({MessID, RemoteJID, Packet}) -> + {integer_to_binary(MessID), jid:to_binary(RemoteJID), exml:to_binary(Packet)}. + %% ---------------------------------------------------------------------- %% Add hooks for mod_mam --spec start_pm(jid:server(), _) -> 'ok'. -start_pm(Host, _Opts) -> +-spec start_hooks(jid:server(), _) -> ok. +start_hooks(Host, _Opts) -> case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of true -> ok; @@ -111,8 +106,8 @@ start_pm(Host, _Opts) -> ok. --spec stop_pm(jid:server()) -> ok. -stop_pm(Host) -> +-spec stop_hooks(jid:server()) -> ok. +stop_hooks(Host) -> case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of true -> ok; @@ -126,587 +121,214 @@ stop_pm(Host) -> ok. %% ---------------------------------------------------------------------- -%% Internal functions and callbacks +%% SQL queries + +register_prepared_queries() -> + prepare_insert(insert_mam_message, 1), + mongoose_rdbms:prepare(mam_archive_remove, mam_message, [user_id], + <<"DELETE FROM mam_message " + "WHERE user_id = ?">>), + mongoose_rdbms:prepare(mam_make_tombstone, mam_message, [message, user_id, id], + <<"UPDATE mam_message SET message = ?, search_body = '' " + "WHERE user_id = ? AND id = ?">>), + {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(1), + mongoose_rdbms:prepare(mam_select_messages_to_retract, mam_message, + [user_id, remote_bare_jid, origin_id, direction], + <<"SELECT ", LimitMSSQL/binary, + " id, message FROM mam_message" + " WHERE user_id = ? AND remote_bare_jid = ? " + " AND origin_id = ? AND direction = ?" + " ORDER BY id DESC ", LimitSQL/binary>>). + +%% ---------------------------------------------------------------------- +%% Declarative logic + +db_mappings() -> + %% One entry per the database field + [#db_mapping{column = id, param = message_id, format = int}, + #db_mapping{column = user_id, param = archive_id, format = int}, + #db_mapping{column = remote_bare_jid, param = remote_jid, format = bare_jid}, + #db_mapping{column = remote_resource, param = remote_jid, format = jid_resource}, + #db_mapping{column = direction, param = direction, format = direction}, + #db_mapping{column = from_jid, param = source_jid, format = jid}, + #db_mapping{column = origin_id, param = origin_id, format = maybe_string}, + #db_mapping{column = message, param = packet, format = xml}, + #db_mapping{column = search_body, param = packet, format = search}]. + +lookup_fields() -> + %% Describe each possible filtering option + [#lookup_field{op = equal, column = user_id, param = archive_id, required = true}, + #lookup_field{op = ge, column = id, param = start_id}, + #lookup_field{op = le, column = id, param = end_id}, + #lookup_field{op = equal, column = remote_bare_jid, param = remote_bare_jid}, + #lookup_field{op = equal, column = remote_resource, param = remote_resource}, + #lookup_field{op = like, column = search_body, param = norm_search_text, value_maker = search_words}]. + +-spec env_vars(jid:lserver(), jid:jid()) -> env_vars(). +env_vars(Host, ArcJID) -> + %% Please, minimize the usage of the host field. + %% It's only for passing into RDBMS. + #{host => Host, + archive_jid => ArcJID, + table => mam_message, + index_hint_fn => fun index_hint_sql/1, + columns_sql_fn => fun columns_sql/1, + column_to_id_fn => fun column_to_id/1, + lookup_fn => fun lookup_query/5, + decode_row_fn => fun row_to_uniform_format/2, + has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam, Host), + has_full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), + db_jid_codec => db_jid_codec(Host, ?MODULE), + db_message_codec => db_message_codec(Host, ?MODULE)}. + +row_to_uniform_format(Row, Env) -> + mam_decoder:decode_row(Row, Env). + +-spec index_hint_sql(env_vars()) -> string(). +index_hint_sql(#{host := Host}) -> + case mongoose_rdbms:db_engine(Host) of + mysql -> "USE INDEX(PRIMARY, i_mam_message_rem) "; + _ -> "" + end. -encode_direction(incoming) -> <<"I">>; -encode_direction(outgoing) -> <<"O">>. +columns_sql(lookup) -> "id, from_jid, message"; +columns_sql(count) -> "COUNT(*)". +%% For each unique column in lookup_fields() +column_to_id(id) -> "i"; +column_to_id(user_id) -> "u"; +column_to_id(remote_bare_jid) -> "b"; +column_to_id(remote_resource) -> "r"; +column_to_id(search_body) -> "s". --spec archive_size(Size :: integer(), Host :: jid:server(), - ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). -archive_size(Size, Host, UserID, _UserJID) when is_integer(Size) -> - {selected, [{BSize}]} = mod_mam_utils:success_sql_execute(Host, mam_archive_size, [UserID]), - mongoose_rdbms:result_to_integer(BSize). +column_names(Mappings) -> + [Column || #db_mapping{column = Column} <- Mappings]. +%% ---------------------------------------------------------------------- +%% Options --spec index_hint_sql(jid:server()) -> string(). -index_hint_sql(Host) -> - case mongoose_rdbms:db_engine(Host) of - mysql -> - "USE INDEX(PRIMARY, i_mam_message_rem) "; - _ -> - "" - end. +-spec db_jid_codec(jid:server(), module()) -> module(). +db_jid_codec(Host, Module) -> + gen_mod:get_module_opt(Host, Module, db_jid_format, mam_jid_mini). +-spec db_message_codec(jid:server(), module()) -> module(). +db_message_codec(Host, Module) -> + gen_mod:get_module_opt(Host, Module, db_message_format, mam_message_compressed_eterm). + +-spec get_retract_id(exml:element(), env_vars()) -> none | binary(). +get_retract_id(Packet, #{has_message_retraction := Enabled}) -> + mod_mam_utils:get_retract_id(Enabled, Packet). + +%% ---------------------------------------------------------------------- +%% Internal functions and callbacks + +-spec archive_size(Size :: integer(), Host :: jid:server(), + ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). +archive_size(Size, Host, ArcID, ArcJID) when is_integer(Size) -> + Filter = [{equal, user_id, ArcID}], + Env = env_vars(Host, ArcJID), + Result = lookup_query(count, Env, Filter, unordered, all), + mongoose_rdbms:selected_to_integer(Result). -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok. -archive_message(_Result, Host, Params) -> +archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> try - do_archive_message(Host, Params) - catch Class:Reason:StackTrace -> + Env = env_vars(Host, ArcJID), + do_archive_message(Host, Params, Env), + retract_message(Host, Params, Env), + ok + catch error:Reason:StackTrace -> ?LOG_ERROR(#{what => archive_message_failed, host => Host, mam_params => Params, - class => Class, reason => Reason, stacktrace => StackTrace}), - {error, Reason} + reason => Reason, stacktrace => StackTrace}), + erlang:raise(error, Reason, StackTrace) end. -do_archive_message(Host, Params) -> - Row = prepare_message(Host, Params), - {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_message, Row), - retract_message(Host, Params). +do_archive_message(Host, Params, Env) -> + Row = mam_encoder:encode_message(Params, Env, db_mappings()), + {updated, 1} = mongoose_rdbms:execute_successfully(Host, insert_mam_message, Row). +%% Retraction logic +%% Called after inserting a new message -spec retract_message(jid:server(), mod_mam:archive_message_params()) -> ok. -retract_message(Host, #{archive_id := UserID, - local_jid := LocJID, - remote_jid := RemJID, - direction := Dir, - packet := Packet}) -> - case mod_mam_utils:get_retract_id(mod_mam, Host, Packet) of +retract_message(Host, #{local_jid := ArcJID} = Params) -> + Env = env_vars(Host, ArcJID), + retract_message(Host, Params, Env). + +-spec retract_message(jid:server(), mod_mam:archive_message_params(), env_vars()) -> ok. +retract_message(Host, #{archive_id := ArcID, remote_jid := RemJID, + direction := Dir, packet := Packet}, Env) -> + case get_retract_id(Packet, Env) of none -> ok; - OriginIDToRetract -> retract_message(Host, UserID, LocJID, RemJID, OriginIDToRetract, Dir) + OriginID -> + Info = get_retraction_info(Host, ArcID, RemJID, OriginID, Dir, Env), + make_tombstone(Host, ArcID, OriginID, Info, Env) end. -retract_message(Host, UserID, LocJID, RemJID, OriginID, Dir) -> - SUserID = use_escaped_integer(escape_user_id(UserID)), - SOriginID = use_escaped_string(escape_string(OriginID)), - SBareRemJID = use_escaped_string(minify_and_escape_bare_jid(Host, LocJID, RemJID)), - SDir = encode_direction(Dir), - Query = query_for_messages_to_retract(SUserID, SBareRemJID, SOriginID, SDir), - {selected, Rows} = mod_mam_utils:success_sql_query(Host, Query), - make_tombstone(Host, SUserID, OriginID, Rows), - ok. - -make_tombstone(_Host, SUserID, OriginID, []) -> +get_retraction_info(Host, ArcID, RemJID, OriginID, Dir, Env) -> + %% Code style notice: + %% - Add Ext prefix for all externally encoded data + %% (in cases, when we usually add Bin, B, S Esc prefixes) + ExtBareRemJID = mam_encoder:encode_jid(jid:to_bare(RemJID), Env), + ExtDir = mam_encoder:encode_direction(Dir), + {selected, Rows} = execute_select_messages_to_retract( + Host, ArcID, ExtBareRemJID, OriginID, ExtDir), + mam_decoder:decode_retraction_info(Env, Rows). + +make_tombstone(_Host, ArcID, OriginID, skip, _Env) -> ?LOG_INFO(#{what => make_tombstone_failed, text => <<"Message to retract was not found by origin id">>, - user_id => SUserID, origin_id => OriginID}); -make_tombstone(Host, SUserID, OriginID, [{ResMessID, ResData}]) -> - Data = mongoose_rdbms:unescape_binary(Host, ResData), - Packet = stored_binary_to_packet(Host, Data), - MessID = mongoose_rdbms:result_to_integer(ResMessID), + user_id => ArcID, origin_id => OriginID}); +make_tombstone(Host, ArcID, OriginID, #{packet := Packet, message_id := MessID}, Env) -> Tombstone = mod_mam_utils:tombstone(Packet, OriginID), - TombstoneData = packet_to_stored_binary(Host, Tombstone), - STombstoneData = mongoose_rdbms:use_escaped_binary( - mongoose_rdbms:escape_binary(Host, TombstoneData)), - BMessID = use_escaped_integer(escape_message_id(MessID)), - UpdateQuery = query_to_make_tombstone(STombstoneData, SUserID, BMessID), - {updated, 1} = mod_mam_utils:success_sql_query(Host, UpdateQuery). - -query_for_messages_to_retract(SUserID, SBareRemJID, SOriginID, SDir) -> - {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits(1), - ["SELECT ", LimitMSSQL, " id, message FROM mam_message" - " WHERE user_id = ", SUserID, " AND remote_bare_jid = ", SBareRemJID, - " AND origin_id = ", SOriginID, " AND direction = '", SDir, "'" - " ORDER BY id DESC ", LimitSQL]. - -query_to_make_tombstone(STombstoneData, SUserID, BMessID) -> - ["UPDATE mam_message SET message = ", STombstoneData, ", search_body = ''" - " WHERE user_id = ", SUserID, " AND id = '", BMessID, "'"]. + TombstoneData = mam_encoder:encode_packet(Tombstone, Env), + execute_make_tombstone(Host, TombstoneData, ArcID, MessID). + +execute_select_messages_to_retract(Host, ArcID, BareRemJID, OriginID, Dir) -> + mongoose_rdbms:execute_successfully(Host, mam_select_messages_to_retract, + [ArcID, BareRemJID, OriginID, Dir]). +execute_make_tombstone(Host, TombstoneData, ArcID, MessID) -> + mongoose_rdbms:execute_successfully(Host, mam_make_tombstone, + [TombstoneData, ArcID, MessID]). + +%% Insert logic -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). -prepare_message(Host, #{message_id := MessID, - archive_id := UserID, - local_jid := LocJID, - remote_jid := RemJID = #jid{lresource = RemLResource}, - source_jid := SrcJID, - origin_id := OriginID, - direction := Dir, - packet := Packet}) -> - SBareRemJID = jid_to_stored_binary(Host, LocJID, jid:to_bare(RemJID)), - SSrcJID = jid_to_stored_binary(Host, LocJID, SrcJID), - SDir = encode_direction(Dir), - SOriginID = case OriginID of - none -> null; - _ -> OriginID - end, - Data = packet_to_stored_binary(Host, Packet), - TextBody = mod_mam_utils:packet_to_search_body(mod_mam, Host, Packet), - [MessID, UserID, SBareRemJID, RemLResource, SDir, SSrcJID, SOriginID, Data, TextBody]. +prepare_message(Host, Params = #{local_jid := ArcJID}) -> + Env = env_vars(Host, ArcJID), + mam_encoder:encode_message(Params, Env, db_mappings()). -spec prepare_insert(Name :: atom(), NumRows :: pos_integer()) -> ok. prepare_insert(Name, NumRows) -> Table = mam_message, - Fields = [id, user_id, remote_bare_jid, remote_resource, - direction, from_jid, origin_id, message, search_body], + Fields = column_names(db_mappings()), {Query, Fields2} = rdbms_queries:create_bulk_insert_query(Table, Fields, NumRows), mongoose_rdbms:prepare(Name, Table, Fields2, Query), ok. --spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> - {ok, mod_mam:lookup_result()}. -lookup_messages({error, _Reason}=Result, _Host, _Params) -> - Result; -lookup_messages(_Result, Host, Params) -> - try - UserID = maps:get(archive_id, Params), - UserJID = maps:get(owner_jid, Params), - RSM = maps:get(rsm, Params), - Borders = maps:get(borders, Params), - Start = maps:get(start_ts, Params), - End = maps:get(end_ts, Params), - Now = maps:get(now, Params), - WithJID = maps:get(with_jid, Params), - SearchText = maps:get(search_text, Params), - PageSize = maps:get(page_size, Params), - IsSimple = maps:get(is_simple, Params), - - do_lookup_messages(Host, - UserID, UserJID, RSM, Borders, - Start, End, Now, WithJID, - mod_mam_utils:normalize_search_text(SearchText), - PageSize, - IsSimple, is_opt_count_supported_for(RSM)) - catch _Type:Reason:S -> - {error, {Reason, {stacktrace, S}}} - end. - -%% Not supported: -%% - #rsm_in{direction = aft, id = ID} -%% - #rsm_in{direction = before, id = ID} -is_opt_count_supported_for(#rsm_in{direction = before, id = undefined}) -> - true; %% last page is supported -is_opt_count_supported_for(#rsm_in{direction = undefined}) -> - true; %% offset -is_opt_count_supported_for(undefined) -> - true; %% no RSM -is_opt_count_supported_for(_) -> - false. - -%% There are several strategies how to extract messages: -%% - we can use regular query that requires counting; -%% - we can reduce number of queries if we skip counting for small data sets; -%% - sometimes we want not to count at all -%% (for example, our client side counts ones and keep the information) -do_lookup_messages(Host, UserID, UserJID, - RSM, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, true, _) -> - %% Simple query without calculating offset and total count - Filter = prepare_filter(Host, UserID, UserJID, Borders, Start, End, WithJID, SearchText), - lookup_messages_simple(Host, UserJID, RSM, PageSize, Filter); -do_lookup_messages(Host, UserID, UserJID, - RSM, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, opt_count, true) -> - %% Extract messages first than calculate offset and total count - %% Useful for small result sets (less than one page, than one query is enough) - Filter = prepare_filter(Host, UserID, UserJID, Borders, Start, End, WithJID, SearchText), - lookup_messages_opt_count(Host, UserJID, RSM, PageSize, Filter); -do_lookup_messages(Host, UserID, UserJID, - RSM, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, _, _) -> - %% Unsupported opt_count or just a regular query - %% Calculate offset and total count first than extract messages - Filter = prepare_filter(Host, UserID, UserJID, Borders, Start, End, WithJID, SearchText), - lookup_messages_regular(Host, UserJID, RSM, PageSize, Filter). - -lookup_messages_simple(Host, UserJID, - #rsm_in{direction = aft, id = ID}, - PageSize, Filter) -> - %% Get last rows from result set - MessageRows = extract_messages(Host, after_id(ID, Filter), 0, PageSize, false), - {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}; -lookup_messages_simple(Host, UserJID, - #rsm_in{direction = before, id = ID}, - PageSize, Filter) -> - MessageRows = extract_messages(Host, before_id(ID, Filter), 0, PageSize, true), - {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}; -lookup_messages_simple(Host, UserJID, - #rsm_in{direction = undefined, index = Offset}, - PageSize, Filter) -> - %% Apply offset - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), - {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}; -lookup_messages_simple(Host, UserJID, undefined, PageSize, Filter) -> - MessageRows = extract_messages(Host, Filter, 0, PageSize, false), - {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}. - -%% Cases that cannot be optimized and used with this function: -%% - #rsm_in{direction = aft, id = ID} -%% - #rsm_in{direction = before, id = ID} -lookup_messages_opt_count(Host, UserJID, - #rsm_in{direction = before, id = undefined}, - PageSize, Filter) -> - %% Last page - MessageRows = extract_messages(Host, Filter, 0, PageSize, true), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {MessageRowsCount, 0, - rows_to_uniform_format(Host, UserJID, MessageRows)}}; - false -> - IndexHintSQL = index_hint_sql(Host), - FirstID = row_to_message_id(hd(MessageRows)), - Offset = calc_count(Host, before_id(FirstID, Filter), IndexHintSQL), - {ok, {Offset + MessageRowsCount, Offset, - rows_to_uniform_format(Host, UserJID, MessageRows)}} - end; -lookup_messages_opt_count(Host, UserJID, - #rsm_in{direction = undefined, index = Offset}, - PageSize, Filter) -> - %% By offset - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {Offset + MessageRowsCount, Offset, - rows_to_uniform_format(Host, UserJID, MessageRows)}}; - false -> - IndexHintSQL = index_hint_sql(Host), - LastID = row_to_message_id(lists:last(MessageRows)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter), IndexHintSQL), - {ok, {Offset + MessageRowsCount + CountAfterLastID, Offset, - rows_to_uniform_format(Host, UserJID, MessageRows)}} - end; -lookup_messages_opt_count(Host, UserJID, - undefined, - PageSize, Filter) -> - %% First page - MessageRows = extract_messages(Host, Filter, 0, PageSize, false), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {MessageRowsCount, 0, - rows_to_uniform_format(Host, UserJID, MessageRows)}}; - false -> - IndexHintSQL = index_hint_sql(Host), - LastID = row_to_message_id(lists:last(MessageRows)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter), IndexHintSQL), - {ok, {MessageRowsCount + CountAfterLastID, 0, - rows_to_uniform_format(Host, UserJID, MessageRows)}} - end. - -lookup_messages_regular(Host, UserJID, - RSM = #rsm_in{direction = aft, id = ID}, - PageSize, Filter) when ID =/= undefined -> - IndexHintSQL = index_hint_sql(Host), - TotalCount = calc_count(Host, Filter, IndexHintSQL), - Offset = calc_offset(Host, Filter, IndexHintSQL, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, from_id(ID, Filter), 0, PageSize + 1, false), - Result = {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}, - mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); -lookup_messages_regular(Host, UserJID, - RSM = #rsm_in{direction = before, id = ID}, - PageSize, Filter) when ID =/= undefined -> - IndexHintSQL = index_hint_sql(Host), - TotalCount = calc_count(Host, Filter, IndexHintSQL), - Offset = calc_offset(Host, Filter, IndexHintSQL, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, to_id(ID, Filter), 0, PageSize + 1, true), - Result = {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}, - mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); -lookup_messages_regular(Host, UserJID, RSM, - PageSize, Filter) -> - IndexHintSQL = index_hint_sql(Host), - TotalCount = calc_count(Host, Filter, IndexHintSQL), - Offset = calc_offset(Host, Filter, IndexHintSQL, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), - {ok, {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}}. - --spec after_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). -after_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id > ", use_escaped_integer(SID)]. - --spec before_id(ID :: escaped_message_id() | undefined, - Filter :: filter()) -> filter(). -before_id(undefined, Filter) -> - Filter; -before_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id < ", use_escaped_integer(SID)]. - --spec from_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). -from_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id >= ", use_escaped_integer(SID)]. - --spec to_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). -to_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id <= ", use_escaped_integer(SID)]. - -rows_to_uniform_format(Host, UserJID, MessageRows) -> - [do_row_to_uniform_format(Host, UserJID, Row) || Row <- MessageRows]. - -do_row_to_uniform_format(Host, UserJID, {BMessID, BSrcJID, SDataRaw}) -> - MessID = mongoose_rdbms:result_to_integer(BMessID), - SrcJID = stored_binary_to_jid(Host, UserJID, BSrcJID), - Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Packet = stored_binary_to_packet(Host, Data), - {MessID, SrcJID, Packet}. - -row_to_message_id({BMessID, _, _}) -> - mongoose_rdbms:result_to_integer(BMessID). - - -%% Removals - +%% Removal logic -spec remove_archive(Acc :: mongoose_acc:t(), Host :: jid:server(), - ArchiveID :: mod_mam:archive_id(), + ArcID :: mod_mam:archive_id(), RoomJID :: jid:jid()) -> mongoose_acc:t(). -remove_archive(Acc, Host, UserID, _UserJID) -> - remove_archive(Host, UserID), +remove_archive(Acc, Host, ArcID, _ArcJID) -> + mongoose_rdbms:execute_successfully(Host, mam_archive_remove, [ArcID]), Acc. -remove_archive(Host, UserID) -> - {updated, _} = - mod_mam_utils:success_sql_query( - Host, - ["DELETE FROM mam_message " - "WHERE user_id = ", use_escaped_integer(escape_user_id(UserID))]). - -%% @doc Each record is a tuple of form -%% `{<<"13663125233">>, <<"bob@localhost">>, <>}'. -%% Columns are `["id", "from_jid", "message"]'. --type msg() :: {binary(), jid:literal_jid(), binary()}. --spec extract_messages(Host :: jid:server(), - Filter :: filter(), IOffset :: non_neg_integer(), IMax :: pos_integer(), - ReverseLimit :: boolean()) -> [msg()]. -extract_messages(_Host, _Filter, _IOffset, 0, _) -> - []; -extract_messages(Host, Filter, IOffset, IMax, false) -> - {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, " ORDER BY id "), - ?LOG_DEBUG(#{what => mam_extract_messages, - mam_filter => Filter, offset => IOffset, max => IMax, - host => Host, message_rows => MessageRows}), - MessageRows; -extract_messages(Host, Filter, IOffset, IMax, true) -> - {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, " ORDER BY id DESC "), - ?LOG_DEBUG(#{what => mam_extract_messages, - mam_filter => Filter, offset => IOffset, max => IMax, - host => Host, message_rows => MessageRows}), - lists:reverse(MessageRows). - -do_extract_messages(Host, Filter, 0, IMax, Order) -> - {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits(IMax), - mod_mam_utils:success_sql_query( - Host, - ["SELECT ", LimitMSSQL, " id, from_jid, message " - "FROM mam_message ", - Filter, - Order, - " ", LimitSQL]); -do_extract_messages(Host, Filter, IOffset, IMax, Order) -> - {LimitSQL, _LimitMSSQL} = rdbms_queries:get_db_specific_limits(IMax), - Offset = rdbms_queries:get_db_specific_offset(IOffset, IMax), - mod_mam_utils:success_sql_query( - Host, - ["SELECT id, from_jid, message " - "FROM mam_message ", - Filter, Order, LimitSQL, Offset]). - -extract_gdpr_messages(Host, ArchiveID) -> - Filter = ["WHERE user_id=", use_escaped_integer(escape_user_id(ArchiveID))], - mod_mam_utils:success_sql_query( - Host, - ["SELECT id, from_jid, message " - "FROM mam_message ", - Filter, " ORDER BY id"]). - -%% @doc Calculate a zero-based index of the row with UID in the result test. -%% -%% If the element does not exists, the ID of the next element will -%% be returned instead. -%% @end -%% "SELECT COUNT(*) as "index" FROM mam_message WHERE id <= '", UID --spec calc_index(Host :: jid:server(), - Filter :: filter(), IndexHintSQL :: string(), - SUID :: escaped_message_id()) -> non_neg_integer(). -calc_index(Host, Filter, IndexHintSQL, SUID) -> - {selected, [{BIndex}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) FROM mam_message ", - IndexHintSQL, Filter, - " AND id <= ", use_escaped_integer(SUID)]), - mongoose_rdbms:result_to_integer(BIndex). - -%% @doc Count of elements in RSet before the passed element. -%% -%% The element with the passed UID can be already deleted. -%% @end -%% "SELECT COUNT(*) as "count" FROM mam_message WHERE id < '", UID --spec calc_before(Host :: jid:server(), - Filter :: filter(), IndexHintSQL :: string(), SUID :: escaped_message_id()) -> - non_neg_integer(). -calc_before(Host, Filter, IndexHintSQL, SUID) -> - {selected, [{BIndex}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) FROM mam_message ", - IndexHintSQL, Filter, - " AND id < ", use_escaped_integer(SUID)]), - mongoose_rdbms:result_to_integer(BIndex). - - -%% @doc Get the total result set size. -%% "SELECT COUNT(*) as "count" FROM mam_message WHERE " --spec calc_count(Host :: jid:server(), - Filter :: filter(), IndexHintSQL :: string()) -> non_neg_integer(). -calc_count(Host, Filter, IndexHintSQL) -> - {selected, [{BCount}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) FROM mam_message ", - IndexHintSQL, Filter]), - mongoose_rdbms:result_to_integer(BCount). - - --spec prepare_filter(Host :: jid:server(), UserID :: mod_mam:archive_id(), - UserJID :: jid:jid(), Borders :: mod_mam:borders(), - Start :: mod_mam:unix_timestamp() | undefined, - End :: mod_mam:unix_timestamp() | undefined, WithJID :: jid:jid(), - SearchText :: binary() | undefined) - -> filter(). -prepare_filter(Host, UserID, UserJID, Borders, Start, End, WithJID, SearchText) -> - {SWithJID, SWithResource} = - case WithJID of - undefined -> {undefined, undefined}; - #jid{lresource = <<>>} -> - {minify_and_escape_bare_jid(Host, UserJID, WithJID), undefined}; - #jid{lresource = WithLResource} -> - {minify_and_escape_bare_jid(Host, UserJID, WithJID), - mongoose_rdbms:escape_string(WithLResource)} - end, - StartID = maybe_encode_compact_uuid(Start, 0), - EndID = maybe_encode_compact_uuid(End, 255), - StartID2 = apply_start_border(Borders, StartID), - EndID2 = apply_end_border(Borders, EndID), - prepare_filter_sql(UserID, StartID2, EndID2, SWithJID, SWithResource, SearchText). - - --spec prepare_filter_sql(UserID :: non_neg_integer(), - StartID :: mod_mam:message_id() | undefined, - EndID :: mod_mam:message_id() | undefined, - SWithJID :: escaped_jid() | undefined, - SWithResource :: escaped_resource() | undefined, - SearchText :: binary() | undefined) -> filter(). -prepare_filter_sql(UserID, StartID, EndID, SWithJID, SWithResource, SearchText) -> - ["WHERE user_id=", use_escaped_integer(escape_user_id(UserID)), - case StartID of - undefined -> ""; - _ -> [" AND id >= ", use_escaped_integer(escape_integer(StartID))] - end, - case EndID of - undefined -> ""; - _ -> [" AND id <= ", use_escaped_integer(escape_integer(EndID))] - end, - case SWithJID of - undefined -> ""; - _ -> [" AND remote_bare_jid = ", use_escaped_string(SWithJID)] - end, - case SWithResource of - undefined -> ""; - _ -> [" AND remote_resource = ", use_escaped_string(SWithResource)] - end, - case SearchText of - undefined -> ""; - _ -> prepare_search_filters(SearchText) - end - ]. - -%% Constructs a separate LIKE filter for each word. -%% SearchText example is "word1%word2%word3". -%% Order of words does not matter (they can go in any order). -prepare_search_filters(SearchText) -> - Words = binary:split(SearchText, <<"%">>, [global]), - [prepare_search_filter(Word) || Word <- Words]. - --spec prepare_search_filter(binary()) -> filter(). -prepare_search_filter(Word) -> - [" AND search_body like ", - %% Search for "%Word%" - mongoose_rdbms:use_escaped_like(mongoose_rdbms:escape_like(Word))]. - -%% @doc #rsm_in{ -%% max = non_neg_integer() | undefined, -%% direction = before | aft | undefined, -%% id = binary() | undefined, -%% index = non_neg_integer() | undefined} --spec calc_offset(Host :: jid:server(), - Filter :: filter(), IndexHintSQL :: string(), PageSize :: non_neg_integer(), - TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in()) -> non_neg_integer(). -calc_offset(_LS, _F, _IH, _PS, _TC, #rsm_in{direction = undefined, index = Index}) - when is_integer(Index) -> - Index; -%% Requesting the Last Page in a Result Set -calc_offset(_LS, _F, _IH, PS, TC, #rsm_in{direction = before, id = undefined}) -> - max(0, TC - PS); -calc_offset(Host, F, IH, PS, _TC, #rsm_in{direction = before, id = ID}) - when is_integer(ID) -> - SID = escape_message_id(ID), - max(0, calc_before(Host, F, IH, SID) - PS); -calc_offset(Host, F, IH, _PS, _TC, #rsm_in{direction = aft, id = ID}) - when is_integer(ID) -> - SID = escape_message_id(ID), - calc_index(Host, F, IH, SID); -calc_offset(_LS, _F, _IH, _PS, _TC, _RSM) -> - 0. - -escape_message_id(MessID) when is_integer(MessID) -> - escape_integer(MessID). - -escape_user_id(UserID) when is_integer(UserID) -> - escape_integer(UserID). - -%% @doc Strip resource, minify and escape JID. -minify_and_escape_bare_jid(Host, LocJID, JID) -> - escape_string(jid_to_stored_binary(Host, LocJID, jid:to_bare(JID))). - -maybe_encode_compact_uuid(undefined, _) -> - undefined; -maybe_encode_compact_uuid(Microseconds, NodeID) -> - encode_compact_uuid(Microseconds, NodeID). +%% GDPR logic +extract_gdpr_messages(Env, ArcID) -> + Filters = [{equal, user_id, ArcID}], + lookup_query(lookup, Env, Filters, asc, all). -%% ---------------------------------------------------------------------- -%% Optimizations - -jid_to_stored_binary(Host, UserJID, JID) -> - Module = db_jid_codec(Host), - mam_jid:encode(Module, UserJID, JID). - -stored_binary_to_jid(Host, UserJID, BSrcJID) -> - Module = db_jid_codec(Host), - mam_jid:decode(Module, UserJID, BSrcJID). - -packet_to_stored_binary(Host, Packet) -> - Module = db_message_codec(Host), - mam_message:encode(Module, Packet). - -stored_binary_to_packet(Host, Bin) -> - Module = db_message_codec(Host), - mam_message:decode(Module, Bin). - --spec db_jid_codec(jid:server()) -> module(). -db_jid_codec(Host) -> - gen_mod:get_module_opt(Host, ?MODULE, db_jid_format, mam_jid_mini). - --spec db_message_codec(jid:server()) -> module(). -db_message_codec(Host) -> - gen_mod:get_module_opt(Host, ?MODULE, db_message_format, mam_message_compressed_eterm). - -%gdpr helpers -gdpr_decode_jid(Host, UserJID, BSrcJID) -> - Codec = mod_mam_meta:get_mam_module_opt(Host, ?MODULE, db_jid_format, mam_jid_mini), - JID = mam_jid:decode(Codec, UserJID, BSrcJID), - jid:to_binary(JID). - -gdpr_decode_packet(Host, SDataRaw) -> - Codec = mod_mam_meta:get_mam_module_opt(Host, ?MODULE, db_message_format, - mam_message_compressed_eterm), - Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Message = mam_message:decode(Codec, Data), - exml:to_binary(Message). +%% Lookup logic +-spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> + {ok, mod_mam:lookup_result()}. +lookup_messages({error, _Reason}=Result, _Host, _Params) -> + Result; +lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> + Env = env_vars(Host, ArcJID), + ExdParams = mam_encoder:extend_lookup_params(Params, Env), + Filter = mam_filter:produce_filter(ExdParams, lookup_fields()), + mam_lookup:lookup(Env, Filter, ExdParams). + +lookup_query(QueryType, Env, Filters, Order, OffsetLimit) -> + mam_lookup_sql:lookup_query(QueryType, Env, Filters, Order, OffsetLimit). diff --git a/src/mam/mod_mam_rdbms_prefs.erl b/src/mam/mod_mam_rdbms_prefs.erl index 9223fa6a73..2173b94796 100644 --- a/src/mam/mod_mam_rdbms_prefs.erl +++ b/src/mam/mod_mam_rdbms_prefs.erl @@ -36,6 +36,7 @@ -spec start(jid:server(), _) -> 'ok'. start(Host, Opts) -> + prepare_queries(Host), case gen_mod:get_module_opt(Host, ?MODULE, pm, false) of true -> start_pm(Host, Opts); @@ -66,6 +67,40 @@ stop(Host) -> end. +%% Prepared queries +prepare_queries(Host) -> + mongoose_rdbms:prepare(mam_prefs_insert, mam_config, [user_id, remote_jid, behaviour], + <<"INSERT INTO mam_config(user_id, remote_jid, behaviour) " + "VALUES (?, ?, ?)">>), + mongoose_rdbms:prepare(mam_prefs_select, mam_config, [user_id], + <<"SELECT remote_jid, behaviour " + "FROM mam_config WHERE user_id=?">>), + mongoose_rdbms:prepare(mam_prefs_select_behaviour, mam_config, + [user_id, remote_jid], + <<"SELECT remote_jid, behaviour " + "FROM mam_config " + "WHERE user_id=? " + "AND (remote_jid='' OR remote_jid=?)">>), + mongoose_rdbms:prepare(mam_prefs_select_behaviour2, mam_config, + [user_id, remote_jid, remote_jid], + <<"SELECT remote_jid, behaviour " + "FROM mam_config " + "WHERE user_id=? " + "AND (remote_jid='' OR remote_jid=? OR remote_jid=?)">>), + OrdBy = order_by_remote_jid_in_delete(Host), + mongoose_rdbms:prepare(mam_prefs_delete, mam_config, [user_id], + <<"DELETE FROM mam_config WHERE user_id=?", OrdBy/binary>>), + ok. + +order_by_remote_jid_in_delete(Host) -> + case mongoose_rdbms:db_engine(Host) of + mysql -> + <<" ORDER BY remote_jid">>; + _ -> + <<"">> + end. + + %% ---------------------------------------------------------------------- %% Add hooks for mod_mam @@ -119,12 +154,7 @@ get_behaviour(DefaultBehaviour, Host, UserID, _LocJID, RemJID) RemLJID = jid:to_lower(RemJID), BRemLBareJID = jid:to_binary(jid:to_bare(RemLJID)), BRemLJID = jid:to_binary(RemLJID), - SRemLBareJID = escape_string(BRemLBareJID), - SRemLJID = escape_string(BRemLJID), - SUserID = escape_integer(UserID), - %% CheckBare if resource is not empty - CheckBare = RemJID#jid.lresource =/= <<>>, - case query_behaviour(Host, SUserID, SRemLJID, SRemLBareJID, CheckBare) of + case query_behaviour(Host, UserID, BRemLJID, BRemLBareJID) of {selected, []} -> DefaultBehaviour; {selected, RemoteJid2Behaviour} -> @@ -160,79 +190,25 @@ set_prefs(_Result, Host, UserID, _ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> {error, Error} end. - -order_by_in_delete(Host) -> - case mongoose_rdbms:db_engine(Host) of - mysql -> - " ORDER BY remote_jid"; - _ -> - "" - end. - set_prefs1(Host, UserID, DefaultMode, AlwaysJIDs, NeverJIDs) -> - %% Lock keys in the same order to avoid deadlock - SUserID = escape_integer(UserID), - EscapedA = escape_string("A"), - EscapedN = escape_string("N"), - JidBehaviourA = [{JID, EscapedA} || JID <- AlwaysJIDs], - JidBehaviourN = [{JID, EscapedN} || JID <- NeverJIDs], - JidBehaviour = lists:keysort(1, JidBehaviourA ++ JidBehaviourN), - ValuesAN = [encode_config_row(SUserID, SBehaviour, escape_string(JID)) - || {JID, SBehaviour} <- JidBehaviour], - SDefaultMode = escape_string(encode_behaviour(DefaultMode)), - DefaultValue = encode_first_config_row(SUserID, SDefaultMode, escape_string("")), - Values = [DefaultValue|ValuesAN], - DelQuery = ["DELETE FROM mam_config WHERE user_id = ", - mongoose_rdbms:use_escaped_integer(SUserID), - order_by_in_delete(Host)], - InsQuery = ["INSERT INTO mam_config(user_id, behaviour, remote_jid) " - "VALUES ", Values], - - run_transaction_or_retry_on_abort(fun() -> - case sql_transaction_map(Host, [DelQuery, InsQuery]) of - {atomic, [{updated, _}, {updated, _}]} -> - {atomic, ok}; - Other -> - Other - end - end, UserID, 5), + Rows = prefs_to_rows(UserID, DefaultMode, AlwaysJIDs, NeverJIDs), + %% MySQL sometimes aborts transaction with reason: + %% "Deadlock found when trying to get lock; try restarting transaction" + mongoose_rdbms:transaction_with_delayed_retry(Host, fun() -> + {updated, _} = + mongoose_rdbms:execute(Host, mam_prefs_delete, [UserID]), + [mongoose_rdbms:execute(Host, mam_prefs_insert, Row) || Row <- Rows], + ok + end, #{user_id => UserID, retries => 5, delay => 100}), ok. -%% Possible error with mysql -%% Reason "Deadlock found when trying to get lock; try restarting transaction" -%% triggered mod_mam_utils:error_on_sql_error -run_transaction_or_retry_on_abort(F, UserID, Retries) -> - Result = F(), - case Result of - {atomic, _} -> - Result; - {aborted, Reason} when Retries > 0 -> - ?LOG_WARNING(#{what => mam_transaction_aborted, - text => <<"Transaction aborted. Restart">>, - user_id => UserID, reason => Reason, retries => Retries}), - timer:sleep(100), - run_transaction_or_retry_on_abort(F, UserID, Retries-1); - _ -> - ?LOG_ERROR(#{what => mam_transaction_failed, - text => <<"Transaction failed. Do not restart">>, - user_id => UserID, reason => Result, retries => Retries}), - erlang:error({transaction_failed, #{user_id => UserID, result => Result}}) - end. - -spec get_prefs(mod_mam:preference(), _Host :: jid:server(), ArchiveID :: mod_mam:archive_id(), ArchiveJID :: jid:jid()) -> mod_mam:preference(). get_prefs({GlobalDefaultMode, _, _}, Host, UserID, _ArcJID) -> - SUserID = escape_integer(UserID), - {selected, Rows} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT remote_jid, behaviour " - "FROM mam_config " - "WHERE user_id=", use_escaped_integer(SUserID)]), + {selected, Rows} = mongoose_rdbms:execute(Host, mam_prefs_select, [UserID]), decode_prefs_rows(Rows, GlobalDefaultMode, [], []). - -spec remove_archive(mongoose_acc:t(), jid:server(), mod_mam:archive_id(), jid:jid()) -> mongoose_acc:t(). remove_archive(Acc, Host, UserID, _ArcJID) -> @@ -240,77 +216,40 @@ remove_archive(Acc, Host, UserID, _ArcJID) -> Acc. remove_archive(Host, UserID) -> - SUserID = escape_integer(UserID), {updated, _} = - mod_mam_utils:success_sql_query( - Host, ["DELETE FROM mam_config WHERE user_id=", use_escaped_integer(SUserID)]). + mongoose_rdbms:execute(Host, mam_prefs_delete, [UserID]). -spec query_behaviour(jid:server(), - SUserID :: mongoose_rdbms:escaped_integer(), - SRemLJID :: mongoose_rdbms:escaped_string(), - SRemLBareJID :: mongoose_rdbms:escaped_string(), - CheckBare :: boolean() + UserID :: non_neg_integer(), + BRemLJID :: binary(), + BRemLBareJID :: binary() ) -> any(). -query_behaviour(Host, SUserID, SRemLJID, SRemLBareJID, CheckBare) -> - Result = - mod_mam_utils:success_sql_query( - Host, - ["SELECT remote_jid, behaviour " - "FROM mam_config " - "WHERE user_id=", use_escaped_integer(SUserID), " " - "AND (remote_jid='' OR remote_jid=", use_escaped_string(SRemLJID), - case CheckBare of - false -> - ""; - true -> - [" OR remote_jid=", use_escaped_string(SRemLBareJID)] - end, - ")"]), - ?LOG_DEBUG(#{what => mam_query_behaviour_result, - user_id => SUserID, result => Result}), - Result. +query_behaviour(Host, UserID, BRemLJID, BRemLJID) -> + mongoose_rdbms:execute(Host, mam_prefs_select_behaviour, + [UserID, BRemLJID]); %% check just bare jid +query_behaviour(Host, UserID, BRemLJID, BRemLBareJID) -> + mongoose_rdbms:execute(Host, mam_prefs_select_behaviour2, + [UserID, BRemLJID, BRemLBareJID]). %% ---------------------------------------------------------------------- %% Helpers --spec encode_behaviour(always | never | roster) -> string(). -encode_behaviour(roster) -> "R"; -encode_behaviour(always) -> "A"; -encode_behaviour(never) -> "N". - +-spec encode_behaviour(always | never | roster) -> binary(). +encode_behaviour(roster) -> <<"R">>; +encode_behaviour(always) -> <<"A">>; +encode_behaviour(never) -> <<"N">>. -spec decode_behaviour(binary()) -> always | never | roster. decode_behaviour(<<"R">>) -> roster; decode_behaviour(<<"A">>) -> always; decode_behaviour(<<"N">>) -> never. --spec encode_first_config_row(SUserID :: mongoose_rdbms:escaped_integer(), - SBehaviour :: mongoose_rdbms:escaped_string(), - SJID :: mongoose_rdbms:escaped_string()) -> - mongoose_rdbms:sql_query_part(). -encode_first_config_row(SUserID, SBehaviour, SJID) -> - ["(", use_escaped_integer(SUserID), - ", ", use_escaped_string(SBehaviour), - ", ", use_escaped_string(SJID), ")"]. - - --spec encode_config_row(SUserID :: mongoose_rdbms:escaped_integer(), - SBehaviour :: mongoose_rdbms:escaped_string(), - SJID :: mongoose_rdbms:escaped_string()) -> - mongoose_rdbms:sql_query_part(). -encode_config_row(SUserID, SBehaviour, SJID) -> - [", (", use_escaped_integer(SUserID), - ", ", use_escaped_string(SBehaviour), - ", ", use_escaped_string(SJID), ")"]. - - --spec sql_transaction_map(jid:server(), [mongoose_rdbms:sql_query()]) -> any(). -sql_transaction_map(LServer, Queries) -> - AtomicF = fun() -> - [mod_mam_utils:success_sql_query(LServer, Query) || Query <- Queries] - end, - mongoose_rdbms:sql_transaction(LServer, AtomicF). - +prefs_to_rows(UserID, DefaultMode, AlwaysJIDs, NeverJIDs) -> + AlwaysRows = [[UserID, JID, encode_behaviour(always)] || JID <- AlwaysJIDs], + NeverRows = [[UserID, JID, encode_behaviour(never)] || JID <- NeverJIDs], + DefaultRow = [UserID, <<>>, encode_behaviour(DefaultMode)], + %% Lock keys in the same order to avoid deadlock + [DefaultRow|lists:sort(AlwaysRows ++ NeverRows)]. -spec decode_prefs_rows([{binary() | jid:jid(), binary()}], DefaultMode :: mod_mam:archive_behaviour(), diff --git a/src/mam/mod_mam_rdbms_user.erl b/src/mam/mod_mam_rdbms_user.erl index 6f5f7c437f..21a6f578c1 100644 --- a/src/mam/mod_mam_rdbms_user.erl +++ b/src/mam/mod_mam_rdbms_user.erl @@ -17,9 +17,6 @@ -export([archive_id/3, remove_archive/4]). -%% gdpr functions --export([get_archive_id/2]). - %% For debugging ONLY -export([create_user_archive/3]). @@ -67,6 +64,7 @@ stop(Host) -> -spec start_pm(jid:server(), _) -> 'ok'. start_pm(Host, _Opts) -> + prepare_queries(), ejabberd_hooks:add(mam_archive_id, Host, ?MODULE, archive_id, 50), case gen_mod:get_module_opt(Host, ?MODULE, auto_remove, false) of true -> @@ -106,7 +104,6 @@ start_muc(Host, _Opts) -> end, ok. - -spec stop_muc(jid:server()) -> 'ok'. stop_muc(Host) -> ejabberd_hooks:delete(mam_muc_archive_id, Host, ?MODULE, archive_id, 50), @@ -119,30 +116,26 @@ stop_muc(Host) -> end, ok. +%% Preparing queries +prepare_queries() -> + mongoose_rdbms:prepare(mam_user_insert, mam_server_user, [server, user_name], + <<"INSERT INTO mam_server_user (server, user_name) VALUES (?, ?)">>), + mongoose_rdbms:prepare(mam_user_select, mam_server_user, [server, user_name], + <<"SELECT id FROM mam_server_user WHERE server=? AND user_name=?">>), + mongoose_rdbms:prepare(mam_user_remove, mam_server_user, [server, user_name], + <<"DELETE FROM mam_server_user WHERE server=? AND user_name=?">>), + ok. %%==================================================================== %% API %%==================================================================== -spec archive_id(undefined | mod_mam:archive_id(), jid:server(), jid:jid()) -> mod_mam:archive_id(). -archive_id(undefined, Host, _ArcJID=#jid{lserver = Server, luser = UserName}) -> - query_archive_id(Host, Server, UserName); +archive_id(undefined, Host, _ArcJID=#jid{lserver = LServer, luser = LUser}) -> + query_archive_id(Host, LServer, LUser); archive_id(ArcID, _Host, _ArcJID) -> ArcID. --spec get_archive_id(jid:server(), jid:user()) -> undefined | mod_mam:archive_id(). -get_archive_id(Host, User) -> - #jid{lserver = Server, luser = UserName} = jid:make(User, Host, <<"">>), - SServer = mongoose_rdbms:escape_string(Server), - SUserName = mongoose_rdbms:escape_string(UserName), - DbType = mongoose_rdbms_type:get(), - case do_query_archive_id(DbType, Host, SServer, SUserName) of - {selected, [{IdBin}]} -> - mongoose_rdbms:result_to_integer(IdBin); - {selected, []} -> - undefined - end. - -spec remove_archive(Acc :: map(), Host :: jid:server(), ArchiveID :: mod_mam:archive_id(), ArchiveJID :: jid:jid()) -> map(). @@ -150,55 +143,38 @@ remove_archive(Acc, Host, ArcID, ArcJID) -> remove_archive(Host, ArcID, ArcJID), Acc. -remove_archive(Host, _ArcID, _ArcJID=#jid{lserver = Server, luser = UserName}) -> - SUserName = mongoose_rdbms:escape_string(UserName), - SServer = mongoose_rdbms:escape_string(Server), +remove_archive(Host, _ArcID, _ArcJID=#jid{lserver = LServer, luser = LUser}) -> {updated, _} = - mongoose_rdbms:sql_query( - Host, - ["DELETE FROM mam_server_user " - "WHERE server = ", mongoose_rdbms:use_escaped_string(SServer), - " AND user_name = ", mongoose_rdbms:use_escaped_string(SUserName)]). + mongoose_rdbms:execute(Host, mam_user_remove, [LUser, LServer]). + %%==================================================================== %% Internal functions %%==================================================================== -spec query_archive_id(jid:server(), jid:lserver(), jid:user()) -> integer(). -query_archive_id(Host, Server, UserName) -> +query_archive_id(Host, LServer, LUser) -> Tries = 5, - query_archive_id(Host, Server, UserName, Tries). + query_archive_id(Host, LServer, LUser, Tries). -query_archive_id(Host, Server, UserName, 0) -> +query_archive_id(Host, LServer, LUser, 0) -> ?LOG_ERROR(#{what => query_archive_id_failed, - host => Host, server => Server, user => UserName}), + host => Host, server => LServer, user => LUser}), error(query_archive_id_failed); -query_archive_id(Host, Server, UserName, Tries) when Tries > 0 -> - SServer = mongoose_rdbms:escape_string(Server), - SUserName = mongoose_rdbms:escape_string(UserName), - DbType = mongoose_rdbms_type:get(), - Result = do_query_archive_id(DbType, Host, SServer, SUserName), - +query_archive_id(Host, LServer, LUser, Tries) when Tries > 0 -> + Result = mongoose_rdbms:execute(Host, mam_user_select, [LServer, LUser]), case Result of {selected, [{IdBin}]} -> mongoose_rdbms:result_to_integer(IdBin); {selected, []} -> %% The user is not found - create_user_archive(Host, Server, UserName), - query_archive_id(Host, Server, UserName, Tries - 1) + create_user_archive(Host, LServer, LUser), + query_archive_id(Host, LServer, LUser, Tries - 1) end. -spec create_user_archive(jid:server(), jid:lserver(), jid:user()) -> ok. -create_user_archive(Host, Server, UserName) -> - SServer = mongoose_rdbms:escape_string(Server), - SUserName = mongoose_rdbms:escape_string(UserName), - Res = - mongoose_rdbms:sql_query( - Host, - ["INSERT INTO mam_server_user " - "(server, user_name) VALUES (", - mongoose_rdbms:use_escaped_string(SServer), ", ", - mongoose_rdbms:use_escaped_string(SUserName), ")"]), +create_user_archive(Host, LServer, LUser) -> + Res = mongoose_rdbms:execute(Host, mam_user_insert, [LServer, LUser]), case Res of {updated, 1} -> ok; @@ -210,17 +186,6 @@ create_user_archive(Host, Server, UserName) -> %% - {error, "[FreeTDS][SQL Server]Violation of UNIQUE KEY constraint" ++ _} %% Let's ignore the errors and just retry in query_archive_id ?LOG_WARNING(#{what => create_user_archive_failed, reason => Res, - user => UserName, host => Host, server => Server}), + user => LUser, host => Host, server => LServer}), ok end. - -do_query_archive_id(mssql, Host, SServer, SUserName) -> - rdbms_queries_mssql:query_archive_id(Host, SServer, SUserName); -do_query_archive_id(_, Host, SServer, SUserName) -> - mongoose_rdbms:sql_query( - Host, - ["SELECT id " - "FROM mam_server_user " - "WHERE server = ", mongoose_rdbms:use_escaped_string(SServer), - " AND user_name = ", mongoose_rdbms:use_escaped_string(SUserName), " " - "LIMIT 1"]). diff --git a/src/mam/mod_mam_utils.erl b/src/mam/mod_mam_utils.erl index 14a2b62545..0053aa9656 100644 --- a/src/mam/mod_mam_utils.erl +++ b/src/mam/mod_mam_utils.erl @@ -29,7 +29,8 @@ get_one_of_path/2, get_one_of_path/3, is_archivable_message/4, - get_retract_id/3, + has_message_retraction/2, + get_retract_id/2, get_origin_id/1, tombstone/2, wrap_message/6, @@ -59,6 +60,7 @@ normalize_search_text/1, normalize_search_text/2, packet_to_search_body/3, + packet_to_search_body/2, has_full_text_search/2 ]). @@ -66,9 +68,6 @@ -export([jid_to_opt_binary/2, expand_minified_jid/2]). -%% SQL --export([success_sql_query/2, success_sql_execute/3]). - %% Other -export([maybe_integer/2, maybe_min/2, @@ -355,11 +354,10 @@ has_chat_marker(Packet) -> _ -> false end. -get_retract_id(Module, Host, Packet) -> - case has_message_retraction(Module, Host) of - true -> get_retract_id(Packet); - false -> none - end. +get_retract_id(true = _Enabled, Packet) -> + get_retract_id(Packet); +get_retract_id(false, _Packet) -> + none. get_retract_id(Packet) -> case exml_query:subelement_with_name_and_ns(Packet, <<"apply-to">>, ?NS_FASTEN) of @@ -761,13 +759,16 @@ normalize_search_text(Text, WordSeparator) -> -spec packet_to_search_body(Module :: mod_mam | mod_mam_muc, Host :: jid:server(), Packet :: exml:element()) -> binary(). packet_to_search_body(Module, Host, Packet) -> - case has_full_text_search(Module, Host) of - true -> - BodyValue = exml_query:path(Packet, [{element, <<"body">>}, cdata], <<>>), - mod_mam_utils:normalize_search_text(BodyValue, <<" ">>); - false -> - <<>> - end. + SearchEnabled = has_full_text_search(Module, Host), + packet_to_search_body(SearchEnabled, Packet). + +-spec packet_to_search_body(Enabled :: boolean(), + Packet :: exml:element()) -> binary(). +packet_to_search_body(true, Packet) -> + BodyValue = exml_query:path(Packet, [{element, <<"body">>}, cdata], <<>>), + mod_mam_utils:normalize_search_text(BodyValue, <<" ">>); +packet_to_search_body(false, _Packet) -> + <<>>. -spec has_full_text_search(Module :: mod_mam | mod_mam_muc, Host :: jid:server()) -> boolean(). has_full_text_search(Module, Host) -> @@ -1089,24 +1090,6 @@ is_jid_in_user_roster(#jid{lserver = LServer} = ToJID, {Subscription, _G} = mongoose_hooks:roster_get_jid_info(LServer, {none, []}, ToJID, RemBareJID), Subscription == from orelse Subscription == both. - --spec success_sql_query(atom() | jid:server(), mongoose_rdbms:sql_query()) -> any(). -success_sql_query(HostOrConn, Query) -> - Result = mongoose_rdbms:sql_query(HostOrConn, Query), - error_on_sql_error(HostOrConn, Query, Result). - --spec success_sql_execute(atom() | jid:server(), atom(), [term()]) -> any(). -success_sql_execute(HostOrConn, Name, Params) -> - Result = mongoose_rdbms:execute(HostOrConn, Name, Params), - error_on_sql_error(HostOrConn, Name, Result). - -error_on_sql_error(HostOrConn, Query, {error, Reason}) -> - ?LOG_ERROR(#{what => mam_sql_error, - host => HostOrConn, query => Query, reason => Reason}), - error({sql_error, Reason}); -error_on_sql_error(_HostOrConn, _Query, Result) -> - Result. - %% @doc Returns a UUIDv4 canonical form binary. -spec wrapper_id() -> binary(). wrapper_id() -> @@ -1144,12 +1127,12 @@ is_policy_violation(TotalCount, Offset, MaxResultLimit, LimitPassed) -> %% return (up to) PageSize messages. %% @end -spec check_for_item_not_found(RSM, PageSize, LookupResult) -> R when - RSM :: jlib:rsm_in(), + RSM :: jlib:rsm_in() | undefined, PageSize :: non_neg_integer(), LookupResult :: mod_mam:lookup_result(), R :: {ok, mod_mam:lookup_result()} | {error, item_not_found}. check_for_item_not_found(#rsm_in{direction = before, id = ID}, - PageSize, {TotalCount, Offset, MessageRows}) when ID =/= undefined -> + PageSize, {TotalCount, Offset, MessageRows}) -> case maybe_last(MessageRows) of {ok, {ID, _, _}} = _IntervalEndpoint -> Page = lists:sublist(MessageRows, PageSize), @@ -1158,7 +1141,7 @@ check_for_item_not_found(#rsm_in{direction = before, id = ID}, {error, item_not_found} end; check_for_item_not_found(#rsm_in{direction = aft, id = ID}, - _PageSize, {TotalCount, Offset, MessageRows0}) when ID =/= undefined -> + _PageSize, {TotalCount, Offset, MessageRows0}) -> case MessageRows0 of [{ID, _, _} = _IntervalEndpoint | MessageRows] -> {ok, {TotalCount, Offset, MessageRows}}; diff --git a/src/mod_last_rdbms.erl b/src/mod_last_rdbms.erl index bb5b8cdf15..96ae404b58 100644 --- a/src/mod_last_rdbms.erl +++ b/src/mod_last_rdbms.erl @@ -25,52 +25,66 @@ remove_user/2]). -spec init(jid:server(), list()) -> ok. -init(_Host, _Opts) -> +init(Host, _Opts) -> + prepare_queries(Host), ok. +%% Prepared query functions +prepare_queries(Host) -> + mongoose_rdbms:prepare(last_select, last, [username], + <<"SELECT seconds, state FROM last WHERE username=?">>), + mongoose_rdbms:prepare(last_count_active, last, [seconds], + <<"SELECT COUNT(*) FROM last WHERE seconds > ?">>), + mongoose_rdbms:prepare(last_delete, last, [username], + <<"delete from last where username=?">>), + rdbms_queries:prepare_upsert(Host, last_upsert, last, + [<<"username">>, <<"seconds">>, <<"state">>], + [<<"seconds">>, <<"state">>], + [<<"username">>]). + +execute_get_last(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, last_select, [LUser]). + +execute_count_active_users(LServer, Seconds) -> + mongoose_rdbms:execute_successfully(LServer, last_count_active, [Seconds]). + +execute_remove_user(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, last_delete, [LUser]). + +execute_upsert_last(Host, LUser, Seconds, State) -> + InsertParams = [LUser, Seconds, State], + UpdateParams = [Seconds, State], + UniqueKeyValues = [LUser], + rdbms_queries:execute_upsert(Host, last_upsert, InsertParams, UpdateParams, UniqueKeyValues). + +%% API functions -spec get_last(jid:luser(), jid:lserver()) -> - {ok, non_neg_integer(), binary()} | {error, term()} | not_found. + {ok, non_neg_integer(), binary()} | not_found. get_last(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - case catch rdbms_queries:get_last(LServer, Username) of - {selected, []} -> - not_found; - {selected, [{STimeStamp, Status}]} -> - case catch mongoose_rdbms:result_to_integer(STimeStamp) of - TimeStamp when is_integer(TimeStamp) -> - {ok, TimeStamp, Status}; - Reason -> - {error, {invalid_timestamp, Reason}} - end; - Reason -> {error, {invalid_result, Reason}} - end. + Result = execute_get_last(LServer, LUser), + decode_last_result(Result). -spec count_active_users(jid:lserver(), non_neg_integer()) -> non_neg_integer(). -count_active_users(LServer, TimeStamp) -> - TimeStampBin = integer_to_binary(TimeStamp), - WhereClause = <<"where seconds > ", TimeStampBin/binary >>, - case rdbms_queries:count_records_where(LServer, <<"last">>, WhereClause) of - {selected, [{Count}]} -> - mongoose_rdbms:result_to_integer(Count); - _ -> - 0 - end. +count_active_users(LServer, Seconds) -> + Result = execute_count_active_users(LServer, Seconds), + mongoose_rdbms:selected_to_integer(Result). -spec set_last_info(jid:luser(), jid:lserver(), - non_neg_integer(), binary()) -> - ok | {error, term()}. -set_last_info(LUser, LServer, TimeStamp, Status) -> - Username = mongoose_rdbms:escape_string(LUser), - Seconds = mongoose_rdbms:escape_integer(TimeStamp), - State = mongoose_rdbms:escape_string(Status), - wrap_rdbms_result(rdbms_queries:set_last_t(LServer, Username, Seconds, State)). + non_neg_integer(), binary()) -> ok | {error, term()}. +set_last_info(LUser, LServer, Seconds, State) -> + wrap_rdbms_result(execute_upsert_last(LServer, LUser, Seconds, State)). -spec remove_user(jid:luser(), jid:lserver()) -> ok | {error, term()}. remove_user(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - wrap_rdbms_result(rdbms_queries:del_last(LServer, Username)). + wrap_rdbms_result(execute_remove_user(LServer, LUser)). + +%% Helper functions +decode_last_result({selected, []}) -> + not_found; +decode_last_result({selected, [{DbSeconds, State}]}) -> + Seconds = mongoose_rdbms:result_to_integer(DbSeconds), + {ok, Seconds, State}. -spec wrap_rdbms_result({error, term()} | any()) -> ok | {error, term()}. wrap_rdbms_result({error, _} = Error) -> Error; wrap_rdbms_result(_) -> ok. - diff --git a/src/mod_privacy_rdbms.erl b/src/mod_privacy_rdbms.erl index 03ed91408c..b4399319be 100644 --- a/src/mod_privacy_rdbms.erl +++ b/src/mod_privacy_rdbms.erl @@ -44,9 +44,71 @@ -include("jlib.hrl"). -include("mod_privacy.hrl"). -init(_Host, _Opts) -> +init(Host, _Opts) -> + prepare_queries(Host), ok. +prepare_queries(Host) -> + %% Queries to privacy_list table + mongoose_rdbms:prepare(privacy_list_get_id, privacy_list, [username, name], + <<"SELECT id FROM privacy_list where username=? AND name=?">>), + mongoose_rdbms:prepare(privacy_list_get_names, privacy_list, [username], + <<"SELECT name FROM privacy_list WHERE username=?">>), + mongoose_rdbms:prepare(privacy_list_delete_by_name, privacy_list, [username, name], + <<"DELETE FROM privacy_list WHERE username=? AND name=?">>), + mongoose_rdbms:prepare(privacy_list_delete_multiple, privacy_list, [username], + <<"DELETE FROM privacy_list WHERE username=?">>), + mongoose_rdbms:prepare(privacy_list_insert, privacy_list, [username, name], + <<"INSERT INTO privacy_list(username, name) VALUES (?, ?)">>), + %% Queries to privacy_default_list table + mongoose_rdbms:prepare(privacy_default_get_name, privacy_default_list, [username], + <<"SELECT name FROM privacy_default_list WHERE username=?">>), + mongoose_rdbms:prepare(privacy_default_delete, privacy_default_list, [username], + <<"DELETE from privacy_default_list WHERE username=?">>), + prepare_default_list_upsert(Host), + %% Queries to privacy_list_data table + mongoose_rdbms:prepare(privacy_data_get_by_id, privacy_list_data, [id], + <<"SELECT ord, t, value, action, match_all, match_iq, " + "match_message, match_presence_in, match_presence_out " + "FROM privacy_list_data " + "WHERE id=? ORDER BY ord">>), + mongoose_rdbms:prepare(delete_data_by_id, privacy_list_data, [id], + <<"DELETE FROM privacy_list_data WHERE id=?">>), + mongoose_rdbms:prepare(privacy_data_delete, privacy_list_data, [id, ord], + <<"DELETE FROM privacy_list_data WHERE id=? AND ord=?">>), + mongoose_rdbms:prepare(privacy_data_update, privacy_list_data, + [t, value, action, match_all, match_iq, + match_message, match_presence_in, match_presence_out, id, ord], + <<"UPDATE privacy_list_data SET " + "t=?, value=?, action=?, match_all=?, match_iq=?, " + "match_message=?, match_presence_in=?, match_presence_out=? " + " WHERE id=? AND ord=?">>), + mongoose_rdbms:prepare(privacy_data_insert, privacy_list_data, + [id, ord, t, value, action, match_all, match_iq, + match_message, match_presence_in, match_presence_out], + <<"INSERT INTO privacy_list_data(" + "id, ord, t, value, action, match_all, match_iq, " + "match_message, match_presence_in, match_presence_out) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)">>), + %% This query uses multiple tables + mongoose_rdbms:prepare(privacy_data_delete_user, privacy_list, [username], + <<"DELETE FROM privacy_list_data WHERE id IN " + "(SELECT id FROM privacy_list WHERE username=?)">>), + ok. + +prepare_default_list_upsert(Host) -> + Fields = [<<"name">>], + Filter = [<<"username">>], + rdbms_queries:prepare_upsert(Host, privacy_default_upsert, privacy_default_list, + Filter ++ Fields, Fields, Filter). + +default_list_upsert(Host, LUser, Name) -> + InsertParams = [LUser, Name], + UpdateParams = [Name], + UniqueKeyValues = [LUser], + rdbms_queries:execute_upsert(Host, privacy_default_upsert, + InsertParams, UpdateParams, UniqueKeyValues). + get_default_list(LUser, LServer) -> case get_default_list_name(LUser, LServer) of none -> @@ -66,7 +128,7 @@ get_list_names(LUser, LServer) -> {ok, {Default, Names}}. get_default_list_name(LUser, LServer) -> - try sql_get_default_privacy_list(LUser, LServer) of + try execute_privacy_default_get_name(LServer, LUser) of {selected, []} -> none; {selected, [{DefName}]} -> @@ -84,7 +146,7 @@ get_default_list_name(LUser, LServer) -> end. get_list_names_only(LUser, LServer) -> - try sql_get_privacy_list_names(LUser, LServer) of + try execute_privacy_list_get_names(LServer, LUser) of {selected, Names} -> [Name || {Name} <- Names]; Other -> @@ -99,9 +161,8 @@ get_list_names_only(LUser, LServer) -> [] end. - get_privacy_list(LUser, LServer, Name) -> - try sql_get_privacy_list_id(LUser, LServer, Name) of + try execute_privacy_list_get_id(LServer, LUser, Name) of {selected, []} -> {error, not_found}; {selected, [{ID}]} -> @@ -121,10 +182,9 @@ get_privacy_list(LUser, LServer, Name) -> end. get_privacy_list_by_id(LUser, LServer, Name, ID, LServer) when is_integer(ID) -> - try sql_get_privacy_list_data_by_id(ID, LServer) of - {selected, RItems} -> - Items = raw_to_items(RItems), - {ok, Items}; + try execute_privacy_data_get_by_id(LServer, ID) of + {selected, Rows} -> + {ok, raw_to_items(Rows)}; Other -> ?LOG_ERROR(#{what => privacy_get_privacy_list_by_id_failed, user => LUser, server => LServer, list_name => Name, list_id => ID, @@ -138,12 +198,9 @@ get_privacy_list_by_id(LUser, LServer, Name, ID, LServer) when is_integer(ID) -> {error, Reason} end. -raw_to_items(RItems) -> - lists:map(fun raw_to_item/1, RItems). - %% @doc Set no default list for user. forget_default_list(LUser, LServer) -> - try sql_unset_default_privacy_list(LUser, LServer) of + try execute_privacy_default_delete(LServer, LUser) of {updated, _} -> ok; Other -> @@ -159,8 +216,8 @@ forget_default_list(LUser, LServer) -> end. set_default_list(LUser, LServer, Name) -> - case rdbms_queries:sql_transaction( - LServer, fun() -> set_default_list_t(LUser, Name) end) of + F = fun() -> set_default_list_t(LServer, LUser, Name) end, + case rdbms_queries:sql_transaction(LServer, F) of {atomic, ok} -> ok; {atomic, {error, Reason}} -> @@ -171,15 +228,14 @@ set_default_list(LUser, LServer, Name) -> {error, Reason} end. --spec set_default_list_t(LUser :: jid:luser(), Name :: binary()) -> ok | {error, not_found}. -set_default_list_t(LUser, Name) -> - case sql_get_privacy_list_names_t(LUser) of +set_default_list_t(LServer, LUser, Name) -> + case execute_privacy_list_get_names(LServer, LUser) of {selected, []} -> {error, not_found}; {selected, Names} -> case lists:member({Name}, Names) of true -> - sql_set_default_privacy_list(LUser, Name), + default_list_upsert(LServer, LUser, Name), ok; false -> {error, not_found} @@ -188,14 +244,12 @@ set_default_list_t(LUser, Name) -> remove_privacy_list(LUser, LServer, Name) -> F = fun() -> - case sql_get_default_privacy_list_t(LUser) of - {selected, []} -> - sql_remove_privacy_list(LUser, Name), - ok; - {selected, [{Name}]} -> + case execute_privacy_default_get_name(LServer, LUser) of + {selected, [{Name}]} -> %% Matches Name variable {error, conflict}; - {selected, [{_Default}]} -> - sql_remove_privacy_list(LUser, Name) + {selected, _} -> + execute_privacy_list_delete_by_name(LServer, LUser, Name), + ok end end, case rdbms_queries:sql_transaction(LServer, F) of @@ -210,82 +264,117 @@ remove_privacy_list(LUser, LServer, Name) -> end. replace_privacy_list(LUser, LServer, Name, List) -> - RItems = lists:map(fun item_to_raw/1, List), + Rows = lists:map(fun item_to_raw/1, List), F = fun() -> - ID = case sql_get_privacy_list_id_t(LUser, Name) of + ResultID = case execute_privacy_list_get_id(LServer, LUser, Name) of {selected, []} -> - sql_add_privacy_list(LUser, Name), - {selected, [{I}]} = - sql_get_privacy_list_id_t(LUser, Name), + execute_privacy_list_insert(LServer, LUser, Name), + {selected, [{I}]} = execute_privacy_list_get_id(LServer, LUser, Name), I; {selected, [{I}]} -> I end, - sql_set_privacy_list(mongoose_rdbms:result_to_integer(ID), RItems), + ID = mongoose_rdbms:result_to_integer(ResultID), + replace_data_rows(LServer, ID, Rows), ok end, - case rdbms_queries:sql_transaction(LServer, F) of - {atomic, ok} -> - ok; - {aborted, Reason} -> - {error, {aborted, Reason}}; - {error, Reason} -> - {error, Reason} - end. + {atomic, ok} = mongoose_rdbms:transaction_with_delayed_retry(LServer, F, #{retries => 5, delay => 100}), + ok. remove_user(LUser, LServer) -> - sql_del_privacy_lists(LUser, LServer). - - -raw_to_item({BType, BValue, BAction, BOrder, BMatchAll, BMatchIQ, - BMatchMessage, BMatchPresenceIn, BMatchPresenceOut}) -> - {Type, Value} = - case BType of - <<"n">> -> - {none, none}; - <<"j">> -> - case jid:from_binary(BValue) of - #jid{} = JID -> - {jid, jid:to_lower(JID)} - end; - <<"g">> -> - {group, BValue}; - <<"s">> -> - case BValue of - <<"none">> -> - {subscription, none}; - <<"both">> -> - {subscription, both}; - <<"from">> -> - {subscription, from}; - <<"to">> -> - {subscription, to} - end - end, - Action = - case BAction of - <<"a">> -> allow; - <<"d">> -> deny; - <<"b">> -> block - end, - Order = mongoose_rdbms:result_to_integer(BOrder), - MatchAll = mongoose_rdbms:to_bool(BMatchAll), - MatchIQ = mongoose_rdbms:to_bool(BMatchIQ), - MatchMessage = mongoose_rdbms:to_bool(BMatchMessage), - MatchPresenceIn = mongoose_rdbms:to_bool(BMatchPresenceIn), - MatchPresenceOut = mongoose_rdbms:to_bool(BMatchPresenceOut), + F = fun() -> remove_user_t(LUser, LServer) end, + rdbms_queries:sql_transaction(LServer, F). + +remove_user_t(LUser, LServer) -> + mongoose_rdbms:execute_successfully(LServer, privacy_data_delete_user, [LUser]), + mongoose_rdbms:execute_successfully(LServer, privacy_list_delete_multiple, [LUser]), + mongoose_rdbms:execute_successfully(LServer, privacy_default_delete, [LUser]). + +execute_privacy_list_get_id(LServer, LUser, Name) -> + mongoose_rdbms:execute_successfully(LServer, privacy_list_get_id, [LUser, Name]). + +execute_privacy_default_get_name(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, privacy_default_get_name, [LUser]). + +execute_privacy_list_get_names(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, privacy_list_get_names, [LUser]). + +execute_privacy_data_get_by_id(LServer, ID) -> + mongoose_rdbms:execute_successfully(LServer, privacy_data_get_by_id, [ID]). + +execute_privacy_default_delete(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, privacy_default_delete, [LUser]). + +execute_privacy_list_delete_by_name(LServer, LUser, Name) -> + mongoose_rdbms:execute_successfully(LServer, privacy_list_delete_by_name, [LUser, Name]). + +execute_privacy_list_insert(LServer, LUser, Name) -> + mongoose_rdbms:execute_successfully(LServer, privacy_list_insert, [LUser, Name]). + +execute_delete_data_by_id(LServer, ID) -> + mongoose_rdbms:execute_successfully(LServer, delete_data_by_id, [ID]). + +replace_data_rows(LServer, ID, []) when is_integer(ID) -> + %% Just remove the data, nothing should be inserted + execute_delete_data_by_id(LServer, ID); +replace_data_rows(LServer, ID, Rows) when is_integer(ID) -> + {selected, OldRows} = execute_privacy_data_get_by_id(LServer, ID), + New = lists:sort(Rows), + Old = lists:sort([tuple_to_list(Row) || Row <- OldRows]), + Diff = diff_rows(ID, New, Old, []), + F = fun({Q, Args}) -> mongoose_rdbms:execute_successfully(LServer, Q, Args) end, + lists:foreach(F, Diff), + ok. + +%% We assume that there are no record duplicates with the same Order. +%% It's checked in the main module for the New argument. +%% It's checked by the database for the Old argument. +diff_rows(ID, [H|New], [H|Old], Ops) -> + diff_rows(ID, New, Old, Ops); %% Not modified +diff_rows(ID, [NewH|NewT] = New, [OldH|OldT] = Old, Ops) -> + NewOrder = hd(NewH), + OldOrder = hd(OldH), + if NewOrder =:= OldOrder -> + Op = {privacy_data_update, tl(NewH) ++ [ID, OldOrder]}, + diff_rows(ID, NewT, OldT, [Op|Ops]); + NewOrder > OldOrder -> + Op = {privacy_data_delete, [ID, OldOrder]}, + diff_rows(ID, New, OldT, [Op|Ops]); + true -> + Op = {privacy_data_insert, [ID|NewH]}, + diff_rows(ID, NewT, Old, [Op|Ops]) + end; +diff_rows(ID, [], [OldH|OldT], Ops) -> + OldOrder = hd(OldH), + Op = {privacy_data_delete, [ID, OldOrder]}, + diff_rows(ID, [], OldT, [Op|Ops]); +diff_rows(ID, [NewH|NewT], [], Ops) -> + Op = {privacy_data_insert, [ID|NewH]}, + diff_rows(ID, NewT, [], [Op|Ops]); +diff_rows(_ID, [], [], Ops) -> + Ops. + +%% Encoding/decoding pure functions + +raw_to_items(Rows) -> + [raw_to_item(Row) || Row <- Rows]. + +raw_to_item({ExtOrder, ExtType, ExtValue, ExtAction, + ExtMatchAll, ExtMatchIQ, ExtMatchMessage, + ExtMatchPresenceIn, ExtMatchPresenceOut}) -> + Type = decode_type(mongoose_rdbms:character_to_integer(ExtType)), #listitem{type = Type, - value = Value, - action = Action, - order = Order, - match_all = MatchAll, - match_iq = MatchIQ, - match_message = MatchMessage, - match_presence_in = MatchPresenceIn, - match_presence_out = MatchPresenceOut - }. - --spec item_to_raw(mod_privacy:list_item()) -> list(mongoose_rdbms:escaped_value()). + value = decode_value(Type, ExtValue), + action = decode_action(mongoose_rdbms:character_to_integer(ExtAction)), + order = mongoose_rdbms:result_to_integer(ExtOrder), + match_all = mongoose_rdbms:to_bool(ExtMatchAll), + match_iq = mongoose_rdbms:to_bool(ExtMatchIQ), + match_message = mongoose_rdbms:to_bool(ExtMatchMessage), + match_presence_in = mongoose_rdbms:to_bool(ExtMatchPresenceIn), + match_presence_out = mongoose_rdbms:to_bool(ExtMatchPresenceOut)}. + +%% Encodes for privacy_data_insert query (but without ID) +-spec item_to_raw(mod_privacy:list_item()) -> list(term()). item_to_raw(#listitem{type = Type, value = Value, action = Action, @@ -294,98 +383,51 @@ item_to_raw(#listitem{type = Type, match_iq = MatchIQ, match_message = MatchMessage, match_presence_in = MatchPresenceIn, - match_presence_out = MatchPresenceOut - }) -> - {BType, BValue} = - case Type of - none -> - {<<"n">>, <<>>}; - jid -> - {<<"j">>, jid:to_binary(Value)}; - group -> - {<<"g">>, Value}; - subscription -> - case Value of - none -> - {<<"s">>, <<"none">>}; - both -> - {<<"s">>, <<"both">>}; - from -> - {<<"s">>, <<"from">>}; - to -> - {<<"s">>, <<"to">>} - end - end, - SType = mongoose_rdbms:escape_string(BType), - SValue = mongoose_rdbms:escape_string(BValue), - BAction = - case Action of - allow -> <<"a">>; - deny -> <<"d">>; - block -> <<"b">> - end, - SAction = mongoose_rdbms:escape_string(BAction), - SOrder = mongoose_rdbms:escape_integer(Order), - SMatchAll = mongoose_rdbms:escape_boolean(MatchAll), - SMatchIQ = mongoose_rdbms:escape_boolean(MatchIQ), - SMatchMessage = mongoose_rdbms:escape_boolean(MatchMessage), - SMatchPresenceIn = mongoose_rdbms:escape_boolean(MatchPresenceIn), - SMatchPresenceOut = mongoose_rdbms:escape_boolean(MatchPresenceOut), - [SType, SValue, SAction, SOrder, SMatchAll, SMatchIQ, - SMatchMessage, SMatchPresenceIn, SMatchPresenceOut]. - -sql_get_default_privacy_list(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:get_default_privacy_list(LServer, Username). - -sql_get_default_privacy_list_t(LUser) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:get_default_privacy_list_t(Username). - -sql_get_privacy_list_names(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:get_privacy_list_names(LServer, Username). - -sql_get_privacy_list_names_t(LUser) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:get_privacy_list_names_t(Username). - -sql_get_privacy_list_id(LUser, LServer, Name) -> - Username = mongoose_rdbms:escape_string(LUser), - SName = mongoose_rdbms:escape_string(Name), - rdbms_queries:get_privacy_list_id(LServer, Username, SName). - -sql_get_privacy_list_id_t(LUser, Name) -> - Username = mongoose_rdbms:escape_string(LUser), - SName = mongoose_rdbms:escape_string(Name), - rdbms_queries:get_privacy_list_id_t(Username, SName). - -sql_get_privacy_list_data_by_id(ID, LServer) when is_integer(ID) -> - rdbms_queries:get_privacy_list_data_by_id(LServer, mongoose_rdbms:escape_integer(ID)). - -sql_set_default_privacy_list(LUser, Name) -> - Username = mongoose_rdbms:escape_string(LUser), - SName = mongoose_rdbms:escape_string(Name), - rdbms_queries:set_default_privacy_list(Username, SName). - -sql_unset_default_privacy_list(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:unset_default_privacy_list(LServer, Username). - -sql_remove_privacy_list(LUser, Name) -> - Username = mongoose_rdbms:escape_string(LUser), - SName = mongoose_rdbms:escape_string(Name), - rdbms_queries:remove_privacy_list(Username, SName). - -sql_add_privacy_list(LUser, Name) -> - Username = mongoose_rdbms:escape_string(LUser), - SName = mongoose_rdbms:escape_string(Name), - rdbms_queries:add_privacy_list(Username, SName). - -sql_set_privacy_list(ID, RItems) when is_integer(ID)-> - rdbms_queries:set_privacy_list(mongoose_rdbms:escape_integer(ID), RItems). - -sql_del_privacy_lists(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - Server = mongoose_rdbms:escape_string(LServer), - rdbms_queries:del_privacy_lists(LServer, Server, Username). + match_presence_out = MatchPresenceOut}) -> + ExtType = encode_type(Type), + ExtValue = encode_value(Type, Value), + ExtAction = encode_action(Action), + Bools = [MatchAll, MatchIQ, MatchMessage, MatchPresenceIn, MatchPresenceOut], + ExtBools = [encode_boolean(X) || X <- Bools], + [Order, ExtType, ExtValue, ExtAction | ExtBools]. + +encode_boolean(true) -> 1; +encode_boolean(false) -> 0. + +encode_action(allow) -> <<"a">>; +encode_action(deny) -> <<"d">>; +encode_action(block) -> <<"b">>. + +decode_action($a) -> allow; +decode_action($d) -> deny; +decode_action($b) -> block. + +encode_subscription(none) -> <<"none">>; +encode_subscription(both) -> <<"both">>; +encode_subscription(from) -> <<"from">>; +encode_subscription(to) -> <<"to">>. + +decode_subscription(<<"none">>) -> none; +decode_subscription(<<"both">>) -> both; +decode_subscription(<<"from">>) -> from; +decode_subscription(<<"to">>) -> to. + +encode_type(none) -> <<"n">>; +encode_type(jid) -> <<"j">>; +encode_type(group) -> <<"g">>; +encode_type(subscription) -> <<"s">>. + +decode_type($n) -> none; +decode_type($j) -> jid; +decode_type($g) -> group; +decode_type($s) -> subscription. + +encode_value(none, _Value) -> <<>>; +encode_value(jid, Value) -> jid:to_binary(Value); +encode_value(group, Value) -> Value; +encode_value(subscription, Value) -> encode_subscription(Value). + +decode_value(none, _) -> none; +decode_value(jid, BinJid) -> jid:to_lower(jid:from_binary(BinJid)); +decode_value(group, Group) -> Group; +decode_value(subscription, ExtSub) -> decode_subscription(ExtSub). diff --git a/src/mod_roster_rdbms.erl b/src/mod_roster_rdbms.erl index 5df5a220c6..b11d80d83d 100644 --- a/src/mod_roster_rdbms.erl +++ b/src/mod_roster_rdbms.erl @@ -13,6 +13,7 @@ -include("mod_roster.hrl"). -include("jlib.hrl"). -include("mongoose.hrl"). +-include("mongoose_logger.hrl"). -behaviour(mod_roster). @@ -36,279 +37,249 @@ -export([raw_to_record/2]). -spec init(jid:server(), list()) -> ok. -init(_Host, _Opts) -> +init(Host, _Opts) -> + prepare_queries(Host), ok. +prepare_queries(Host) -> + mongoose_rdbms:prepare(roster_group_insert, rostergroups, [username, jid, grp], + <<"INSERT INTO rostergroups(username, jid, grp) " + "VALUES (?, ?, ?)">>), + mongoose_rdbms:prepare(roster_version_get, roster_version, [username], + <<"SELECT version FROM roster_version " + "WHERE username=?">>), + mongoose_rdbms:prepare(roster_get, rosterusers, [username], + <<"SELECT ", (roster_fields())/binary, + " FROM rosterusers WHERE username=?">>), + mongoose_rdbms:prepare(roster_get_by_jid, rostergroups, [username, jid], + <<"SELECT ", (roster_fields())/binary, + " FROM rosterusers WHERE username=? AND jid=?">>), + mongoose_rdbms:prepare(roster_group_get, rostergroups, [username], + <<"SELECT jid, grp FROM rostergroups WHERE username=?">>), + mongoose_rdbms:prepare(roster_group_get_by_jid, rostergroups, [username, jid], + <<"SELECT grp FROM rostergroups " + "WHERE username=? AND jid=?">>), + mongoose_rdbms:prepare(roster_delete, rosterusers, [username], + <<"DELETE FROM rosterusers WHERE username=?">>), + mongoose_rdbms:prepare(roster_group_delete, rostergroups, [username], + <<"DELETE FROM rostergroups WHERE username=?">>), + mongoose_rdbms:prepare(roster_delete_by_jid, rosterusers, [username, jid], + <<"DELETE FROM rosterusers WHERE username=? AND jid=?">>), + mongoose_rdbms:prepare(roster_group_delete_by_jid, rostergroups, [username, jid], + <<"DELETE FROM rostergroups WHERE username=? AND jid=?">>), + prepare_roster_upsert(Host), + prepare_version_upsert(Host), + ok. + + +%% We don't care about `server, subscribe, type' fields +roster_fields() -> + <<"username, jid, nick, subscription, ask, askmessage">>. + +prepare_roster_upsert(Host) -> + Fields = [<<"nick">>, <<"subscription">>, <<"ask">>, + <<"askmessage">>, <<"server">>, <<"subscribe">>, <<"type">>], + Filter = [<<"username">>, <<"jid">>], + rdbms_queries:prepare_upsert(Host, roster_upsert, rosterusers, + Filter ++ Fields, Fields, Filter). + +prepare_version_upsert(Host) -> + Fields = [<<"version">>], + Filter = [<<"username">>], + rdbms_queries:prepare_upsert(Host, roster_version_upsert, roster_version, + Filter ++ Fields, Fields, Filter). + +%% Query Helpers + +execute_roster_get(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, roster_get, [LUser]). + +roster_upsert(Host, LUser, BinJID, RosterRow) -> + [_LUser, _BinJID|Rest] = RosterRow, + InsertParams = RosterRow, + UpdateParams = Rest, + UniqueKeyValues = [LUser, BinJID], + rdbms_queries:execute_upsert(Host, roster_upsert, InsertParams, UpdateParams, UniqueKeyValues). + +version_upsert(Host, LUser, Version) -> + InsertParams = [LUser, Version], + UpdateParams = [Version], + UniqueKeyValues = [LUser], + rdbms_queries:execute_upsert(Host, roster_version_upsert, InsertParams, UpdateParams, UniqueKeyValues). + +%% API functions + -spec transaction(LServer :: jid:lserver(), F :: fun()) -> {aborted, Reason :: any()} | {atomic, Result :: any()}. transaction(LServer, F) -> mongoose_rdbms:sql_transaction(LServer, F). --spec read_roster_version(jid:luser(), jid:lserver()) --> binary() | error. +-spec read_roster_version(jid:luser(), jid:lserver()) -> binary() | error. read_roster_version(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - case rdbms_queries:get_roster_version(LServer, Username) - of + case mongoose_rdbms:execute_successfully(LServer, roster_version_get, [LUser]) of {selected, [{Version}]} -> Version; {selected, []} -> error end. -write_roster_version(LUser, LServer, InTransaction, Ver) -> - Username = mongoose_rdbms:escape_string(LUser), - EVer = mongoose_rdbms:escape_string(Ver), - case InTransaction of - true -> - rdbms_queries:set_roster_version(Username, EVer); - _ -> - rdbms_queries:sql_transaction(LServer, - fun () -> - rdbms_queries:set_roster_version(Username, - EVer) - end) - end. +write_roster_version(LUser, LServer, _InTransaction, Ver) -> + version_upsert(LServer, LUser, Ver). get_roster(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - try rdbms_queries:get_roster(LServer, Username) of - {selected, Items} when is_list(Items) -> - {selected, JIDGroups} = rdbms_queries:get_roster_jid_groups(LServer, Username), - GroupsDict = lists:foldl(fun ({J, G}, Acc) -> - dict:append(J, G, Acc) - end, - dict:new(), JIDGroups), - RItems = lists:flatmap(fun (I) -> - raw_to_record_with_group(LServer, I, GroupsDict) - end, - Items), - RItems; - _ -> [] - catch Class:Reason:StackTrace -> - ?LOG_ERROR(#{what => get_roster_failed, class => Class, reason => Reason, - stacktrace => StackTrace, user => LUser, host => LServer}), - [] - end. + {selected, Rows} = execute_roster_get(LServer, LUser), + {selected, GroupRows} = mongoose_rdbms:execute_successfully(LServer, roster_group_get, [LUser]), + decode_roster_rows(LServer, Rows, GroupRows). -raw_to_record_with_group(LServer, I, GroupsDict) -> - case raw_to_record(LServer, I) of - %% Bad JID in database: - error -> []; - R -> - SJID = jid:to_binary(R#roster.jid), - Groups = case dict:find(SJID, GroupsDict) of - {ok, Gs} -> Gs; - error -> [] - end, - [R#roster{groups = Groups}] - end. +get_roster_entry_t(LUser, LServer, LJID) -> + %% `mongoose_rdbms:execute' automatically detects if we are in a transaction or not. + get_roster_entry(LUser, LServer, LJID). get_roster_entry(LUser, LServer, LJID) -> - do_get_roster_entry(LUser, LServer, LJID, get_roster_by_jid). - -get_roster_entry_t(LUser, LServer, LJID) -> - do_get_roster_entry(LUser, LServer, LJID, get_roster_by_jid_t). - -do_get_roster_entry(LUser, LServer, LJID, FuncName) -> - Username = mongoose_rdbms:escape_string(LUser), - SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - {selected, Res} = case FuncName of - get_roster_by_jid -> - rdbms_queries:get_roster_by_jid(LServer, Username, SJID); - get_roster_by_jid_t -> - rdbms_queries:get_roster_by_jid_t(LServer, Username, SJID) - end, - case Res of - [] -> - does_not_exist; - [I] -> - R = raw_to_record(LServer, I), - case R of - %% Bad JID in database: - error -> - #roster{usj = {LUser, LServer, LJID}, - us = {LUser, LServer}, jid = LJID}; - _ -> - R#roster{usj = {LUser, LServer, LJID}, - us = {LUser, LServer}, jid = LJID} - end + BinJID = jid:to_binary(LJID), + {selected, Rows} = mongoose_rdbms:execute_successfully(LServer, roster_get_by_jid, [LUser, BinJID]), + case Rows of + [] -> does_not_exist; + [Row] -> row_to_record(LServer, Row) end. +get_roster_entry_t(LUser, LServer, LJID, full) -> + get_roster_entry(LUser, LServer, LJID, full). + +%% full means we should query for groups too get_roster_entry(LUser, LServer, LJID, full) -> case get_roster_entry(LUser, LServer, LJID) of does_not_exist -> does_not_exist; - Rentry -> - case read_subscription_and_groups(LUser, LServer, LJID) of - error -> error; - {Subscription, Groups} -> - Rentry#roster{subscription = Subscription, groups = Groups} - end - end. - -get_roster_entry_t(LUser, LServer, LJID, full) -> - case get_roster_entry_t(LUser, LServer, LJID) of - does_not_exist -> does_not_exist; - Rentry -> - case read_subscription_and_groups_t(LUser, LServer, LJID) of - error -> error; - {Subscription, Groups} -> - Rentry#roster{subscription = Subscription, groups = Groups} - end + Rec -> + Groups = get_groups_by_jid(LUser, LServer, LJID), + record_with_groups(Rec, Groups) end. - get_subscription_lists(_, LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - try rdbms_queries:get_roster(LServer, Username) of - {selected, Items} when is_list(Items) -> - Items; - Other -> - ?LOG_ERROR(#{what => get_subscription_lists_failed, reason => Other, - user => LUser, host => LServer}), - [] - catch Class:Reason:StackTrace -> - ?LOG_ERROR(#{what => get_subscription_lists_failed, class => Class, - reason => Reason, stacktrace => StackTrace, - user => LUser, host => LServer}), - [] - end. + {selected, Rows} = execute_roster_get(LServer, LUser), + [row_to_record(LServer, Row) || Row <- Rows]. roster_subscribe_t(LUser, LServer, LJID, Item) -> - ItemVals = record_to_string(Item), - Username = mongoose_rdbms:escape_string(LUser), - SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - rdbms_queries:roster_subscribe(LServer, Username, SJID, - ItemVals). + BinJID = jid:to_binary(LJID), + RosterRow = record_to_row(Item), + roster_upsert(LServer, LUser, BinJID, RosterRow). remove_user(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:del_user_roster_t(LServer, Username), + F = fun() -> + mongoose_rdbms:execute_successfully(LServer, roster_delete, [LUser]), + mongoose_rdbms:execute_successfully(LServer, roster_group_delete, [LUser]) + end, + mongoose_rdbms:sql_transaction(LServer, F), ok. update_roster_t(LUser, LServer, LJID, Item) -> - Username = mongoose_rdbms:escape_string(LUser), - SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - ItemVals = record_to_string(Item), - ItemGroups = groups_to_string(Item), - rdbms_queries:update_roster(LServer, Username, SJID, ItemVals, ItemGroups). + BinJID = jid:to_binary(LJID), + RosterRow = record_to_row(Item), + GroupRows = groups_to_rows(Item), + roster_upsert(LServer, LUser, BinJID, RosterRow), + mongoose_rdbms:execute_successfully(LServer, roster_group_delete_by_jid, [LUser, BinJID]), + [mongoose_rdbms:execute_successfully(LServer, roster_group_insert, GroupRow) + || GroupRow <- GroupRows], + ok. del_roster_t(LUser, LServer, LJID) -> - Username = mongoose_rdbms:escape_string(LUser), - SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - rdbms_queries:del_roster(LServer, Username, SJID). - -raw_to_record(LServer, - {User, SJID, Nick, SSubscription, SAsk, SAskMessage, - _SServer, _SSubscribe, _SType}) -> - case jid:from_binary(SJID) of - error -> error; - JID -> - LJID = jid:to_lower(JID), - Subscription = case SSubscription of - <<"B">> -> both; - <<"T">> -> to; - <<"F">> -> from; - _ -> none - end, - Ask = case SAsk of - <<"S">> -> subscribe; - <<"U">> -> unsubscribe; - <<"B">> -> both; - <<"O">> -> out; - <<"I">> -> in; - _ -> none - end, - #roster{usj = {User, LServer, LJID}, - us = {User, LServer}, jid = LJID, name = Nick, - subscription = Subscription, ask = Ask, - askmessage = SAskMessage} - end. - + BinJID = jid:to_binary(LJID), + mongoose_rdbms:execute_successfully(LServer, roster_delete_by_jid, [LUser, BinJID]), + mongoose_rdbms:execute_successfully(LServer, roster_group_delete_by_jid, [LUser, BinJID]). -read_subscription_and_groups(LUser, LServer, LJID) -> - read_subscription_and_groups(LUser, LServer, LJID, get_subscription, - get_rostergroup_by_jid). - -read_subscription_and_groups_t(LUser, LServer, LJID) -> - read_subscription_and_groups(LUser, LServer, LJID, get_subscription_t, - get_rostergroup_by_jid_t). - -read_subscription_and_groups(LUser, LServer, LJID, GSFunc, GRFunc) -> - Username = mongoose_rdbms:escape_string(LUser), - SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - SubResult = case GSFunc of - get_subscription -> - catch rdbms_queries:get_subscription(LServer, Username, SJID); - get_subscription_t -> - catch rdbms_queries:get_subscription_t(LServer, Username, SJID) - end, - case SubResult of - {selected, [{SSubscription}]} -> - Subscription = case SSubscription of - <<"B">> -> both; - <<"T">> -> to; - <<"F">> -> from; - _ -> none - end, - GRResult = case GRFunc of - get_rostergroup_by_jid -> - catch rdbms_queries:get_rostergroup_by_jid(LServer, Username, SJID); - get_rostergroup_by_jid_t -> - catch rdbms_queries:get_rostergroup_by_jid_t(LServer, Username, SJID) - end, - Groups = case GRResult of - {selected, JGrps} when is_list(JGrps) -> - [JGrp || {JGrp} <- JGrps]; - _ -> - ?LOG_ERROR(#{what => read_subscription_and_groups_failed, - reason => GRResult, user => LUser, - host => LServer}), - [] - end, - {Subscription, Groups}; - E -> - ?LOG_ERROR(#{what => read_subscription_and_groups_failed, - reason => E, user => LUser, host => LServer}), - error - end. +get_groups_by_jid(LUser, LServer, LJID) -> + BinJID = jid:to_binary(LJID), + {selected, Rows} = mongoose_rdbms:execute_successfully( + LServer, roster_group_get_by_jid, [LUser, BinJID]), + [Group || {Group} <- Rows]. %%============================================================================== %% Helper functions %%============================================================================== -record_to_string(#roster{us = {User, _Server}, - jid = JID, name = Name, subscription = Subscription, - ask = Ask, askmessage = AskMessage}) -> - Username = mongoose_rdbms:escape_string(User), - SJID = - mongoose_rdbms:escape_string(jid:to_binary(jid:to_lower(JID))), - Nick = mongoose_rdbms:escape_string(Name), - SSubscription = mongoose_rdbms:escape_string(case Subscription of - both -> <<"B">>; - to -> <<"T">>; - from -> <<"F">>; - none -> <<"N">> - end), - SAsk = mongoose_rdbms:escape_string(case Ask of - subscribe -> <<"S">>; - unsubscribe -> <<"U">>; - both -> <<"B">>; - out -> <<"O">>; - in -> <<"I">>; - none -> <<"N">> - end), - SAskMessage = mongoose_rdbms:escape_string(AskMessage), - [Username, SJID, Nick, SSubscription, SAsk, SAskMessage, - mongoose_rdbms:escape_string(<<"N">>), - mongoose_rdbms:escape_string(<<"">>), - mongoose_rdbms:escape_string(<<"item">>)]. - -groups_to_string(#roster{us = {User, _Server}, - jid = JID, groups = Groups}) -> - Username = mongoose_rdbms:escape_string(User), - SJID = mongoose_rdbms:escape_string(jid:to_binary(jid:to_lower(JID))), - lists:foldl(fun (<<"">>, Acc) -> Acc; - (Group, Acc) -> - G = mongoose_rdbms:escape_string(Group), - [[Username, SJID, G] | Acc] - end, - [], Groups). +decode_subscription($B) -> both; +decode_subscription($T) -> to; +decode_subscription($F) -> from; +decode_subscription($N) -> none. + +encode_subscription(both) -> <<"B">>; +encode_subscription(to) -> <<"T">>; +encode_subscription(from) -> <<"F">>; +encode_subscription(none) -> <<"N">>. + +decode_ask($S) -> subscribe; +decode_ask($U) -> unsubscribe; +decode_ask($B) -> both; +decode_ask($O) -> out; +decode_ask($I) -> in; +decode_ask($N) -> none. + +encode_ask(subscribe) -> <<"S">>; +encode_ask(unsubscribe) -> <<"U">>; +encode_ask(both) -> <<"B">>; +encode_ask(out) -> <<"O">>; +encode_ask(in) -> <<"I">>; +encode_ask(none) -> <<"N">>. + +record_to_row(#roster{us = {LUser, _Server}, + jid = JID, name = Nick, subscription = Subscription, + ask = Ask, askmessage = AskMessage}) -> + BinJID = jid:to_binary(jid:to_lower(JID)), + ExtSubscription = encode_subscription(Subscription), + ExtAsk = encode_ask(Ask), + [LUser, BinJID, Nick, ExtSubscription, ExtAsk, AskMessage, + <<"N">>, <<>>, <<"item">>]. + +groups_to_rows(#roster{us = {LUser, _LServer}, jid = JID, groups = Groups}) -> + BinJID = jid:to_binary(jid:to_lower(JID)), + lists:foldl(fun (<<>>, Acc) -> Acc; + (Group, Acc) -> [[LUser, BinJID, Group] | Acc] + end, [], Groups). + +%% We must not leak our external RDBMS format representation into MongooseIM +raw_to_record(_LServer, Rec) -> Rec. + +%% Decode fields from `roster_fields()' into a record +row_to_record(LServer, + {User, BinJID, Nick, ExtSubscription, ExtAsk, AskMessage}) -> + JID = parse_jid(BinJID), + LJID = jid:to_lower(JID), %% Convert to tuple {U,S,R} + Subscription = decode_subscription(mongoose_rdbms:character_to_integer(ExtSubscription)), + Ask = decode_ask(mongoose_rdbms:character_to_integer(ExtAsk)), + USJ = {User, LServer, LJID}, + US = {User, LServer}, + #roster{usj = USJ, us = US, jid = LJID, name = Nick, + subscription = Subscription, ask = Ask, askmessage = AskMessage}. + +row_to_binary_jid(Row) -> element(2, Row). + +record_with_groups(Rec, Groups) -> + Rec#roster{groups = Groups}. + +%% We require all DB jids to be parsable. +%% They should be lowered too. +parse_jid(BinJID) -> + case jid:from_binary(BinJID) of + error -> + error(#{what => parse_jid_failed, jid => BinJID}); + JID -> + JID + end. + +decode_roster_rows(LServer, Rows, JIDGroups) -> + GroupsDict = pairs_to_dict(JIDGroups), + [raw_to_record_with_group(LServer, Row, GroupsDict) || Row <- Rows]. + +pairs_to_dict(Pairs) -> + F = fun ({K, V}, Acc) -> dict:append(K, V, Acc) end, + lists:foldl(F, dict:new(), Pairs). + +raw_to_record_with_group(LServer, Row, GroupsDict) -> + Rec = row_to_record(LServer, Row), + BinJID = row_to_binary_jid(Row), + Groups = dict_get(BinJID, GroupsDict, []), + record_with_groups(Rec, Groups). + +dict_get(K, Dict, Default) -> + case dict:find(K, Dict) of + {ok, Values} -> Values; + error -> Default + end. diff --git a/src/muc_light/mod_muc_light_db.erl b/src/muc_light/mod_muc_light_db.erl index 5cc207492b..d6a86f5d96 100644 --- a/src/muc_light/mod_muc_light_db.erl +++ b/src/muc_light/mod_muc_light_db.erl @@ -55,9 +55,6 @@ -callback get_config(RoomUS :: jid:simple_bare_jid()) -> {ok, mod_muc_light_room_config:kv(), Version :: binary()} | {error, not_exists}. --callback get_config(RoomUS :: jid:simple_bare_jid(), Key :: atom()) -> - {ok, term(), Version :: binary()} | {error, not_exists | invalid_opt}. - -callback set_config(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), Version :: binary()) -> {ok, PrevVersion :: binary()} | {error, not_exists}. diff --git a/src/muc_light/mod_muc_light_db_mnesia.erl b/src/muc_light/mod_muc_light_db_mnesia.erl index 09eb86122c..6451245bd6 100644 --- a/src/muc_light/mod_muc_light_db_mnesia.erl +++ b/src/muc_light/mod_muc_light_db_mnesia.erl @@ -36,7 +36,6 @@ remove_user/2, get_config/1, - get_config/2, set_config/3, set_config/4, @@ -142,19 +141,6 @@ get_config(RoomUS) -> [#muc_light_room{ config = Config, version = Version }] -> {ok, Config, Version} end. --spec get_config(RoomUS :: jid:simple_bare_jid(), Key :: atom()) -> - {ok, term(), Version :: binary()} | {error, not_exists | invalid_opt}. -get_config(RoomUS, Option) -> - case get_config(RoomUS) of - {ok, Config, Version} -> - case lists:keyfind(Option, 1, Config) of - {_, Value} -> {ok, Value, Version}; - false -> {error, invalid_opt} - end; - Error -> - Error - end. - -spec set_config(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), Version :: binary()) -> diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index 71d1050529..ff90bf56a9 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -36,7 +36,6 @@ remove_user/2, get_config/1, - get_config/2, set_config/3, set_config/4, @@ -50,17 +49,13 @@ get_info/1 ]). -%% Conversions --export([ - what_db2atom/1, what_atom2db/1, - aff_db2atom/1, aff_atom2db/1 - ]). - %% Extra API for testing -export([ force_clear/0 ]). +-type room_id() :: non_neg_integer(). + -include("mongoose.hrl"). -include("jlib.hrl"). -include("mod_muc_light.hrl"). @@ -72,53 +67,349 @@ %% ------------------------ Backend start/stop ------------------------ -spec start(Host :: jid:server(), MUCHost :: jid:server()) -> ok. -start(_Host, _MUCHost) -> +start(Host, _MUCHost) -> + prepare_queries(Host), ok. -spec stop(Host :: jid:server(), MUCHost :: jid:server()) -> ok. stop(_Host, _MUCHost) -> ok. +%% ------------------------ SQL ------------------------------------------- + +prepare_queries(Host) -> + prepare_cleaning_queries(Host), + prepare_room_queries(Host), + prepare_affiliation_queries(Host), + prepare_config_queries(Host), + prepare_blocking_queries(Host), + ok. + +prepare_cleaning_queries(_Host) -> + mongoose_rdbms:prepare(muc_light_config_delete_all, muc_light_config, [], + <<"DELETE FROM muc_light_config">>), + mongoose_rdbms:prepare(muc_light_occupants_delete_all, muc_light_occupants, [], + <<"DELETE FROM muc_light_occupants">>), + mongoose_rdbms:prepare(muc_light_rooms_delete_all, muc_light_rooms, [], + <<"DELETE FROM muc_light_rooms">>), + mongoose_rdbms:prepare(muc_light_blocking_delete_all, muc_light_blocking, [], + <<"DELETE FROM muc_light_blocking">>), + ok. + +prepare_room_queries(_Host) -> + %% Returns maximum 1 record + mongoose_rdbms:prepare(muc_light_select_room_id, muc_light_rooms, + [luser, lserver], + <<"SELECT id FROM muc_light_rooms " + "WHERE luser = ? AND lserver = ?">>), + mongoose_rdbms:prepare(muc_light_select_room_id_and_version, muc_light_rooms, + [luser, lserver], + <<"SELECT id, version FROM muc_light_rooms " + "WHERE luser = ? AND lserver = ?">>), + mongoose_rdbms:prepare(muc_light_insert_room, muc_light_rooms, + [luser, lserver, version], + <<"INSERT INTO muc_light_rooms (luser, lserver, version)" + " VALUES (?, ?, ?)">>), + mongoose_rdbms:prepare(muc_light_update_room_version, muc_light_rooms, + [luser, lserver, version], + <<"UPDATE muc_light_rooms SET version = ? " + " WHERE luser = ? AND lserver = ?">>), + mongoose_rdbms:prepare(muc_light_delete_room, muc_light_rooms, + [luser, lserver], + <<"DELETE FROM muc_light_rooms" + " WHERE luser = ? AND lserver = ?">>), + %% These queries use multiple tables + mongoose_rdbms:prepare(muc_light_select_user_rooms, muc_light_occupants, + [luser, lserver], + <<"SELECT r.luser, r.lserver " + " FROM muc_light_occupants AS o " + " INNER JOIN muc_light_rooms AS r ON o.room_id = r.id" + " WHERE o.luser = ? AND o.lserver = ?">>), + mongoose_rdbms:prepare(muc_light_select_user_rooms_count, muc_light_occupants, + [luser, lserver], + <<"SELECT count(*) " + " FROM muc_light_occupants AS o " + " INNER JOIN muc_light_rooms AS r ON o.room_id = r.id" + " WHERE o.luser = ? AND o.lserver = ?">>), + ok. + +prepare_affiliation_queries(_Host) -> + %% This query uses multiple tables + %% Also returns a room version + mongoose_rdbms:prepare(muc_light_select_affs_by_us, muc_light_rooms, + [luser, lserver], + <<"SELECT version, o.luser, o.lserver, aff" + " FROM muc_light_rooms AS r " + " LEFT OUTER JOIN muc_light_occupants AS o ON r.id = o.room_id" + " WHERE r.luser = ? AND r.lserver = ?">>), + mongoose_rdbms:prepare(muc_light_select_affs_by_room_id, muc_light_occupants, + [room_id], + <<"SELECT luser, lserver, aff " + "FROM muc_light_occupants WHERE room_id = ?">>), + mongoose_rdbms:prepare(muc_light_insert_aff, muc_light_occupants, + [room_id, luser, lserver, aff], + <<"INSERT INTO muc_light_occupants" + " (room_id, luser, lserver, aff)" + " VALUES(?, ?, ?, ?)">>), + mongoose_rdbms:prepare(muc_light_update_aff, muc_light_occupants, + [aff, room_id, luser, lserver], + <<"UPDATE muc_light_occupants SET aff = ? " + "WHERE room_id = ? AND luser = ? AND lserver = ?">>), + mongoose_rdbms:prepare(muc_light_delete_affs, muc_light_occupants, + [room_id], + <<"DELETE FROM muc_light_occupants WHERE room_id = ?">>), + mongoose_rdbms:prepare(muc_light_delete_aff, muc_light_occupants, + [room_id, luser, lserver], + <<"DELETE FROM muc_light_occupants " + "WHERE room_id = ? AND luser = ? AND lserver = ?">>), + ok. + +prepare_config_queries(_Host) -> + mongoose_rdbms:prepare(muc_light_select_config_by_room_id, muc_light_config, + [room_id], + <<"SELECT opt, val FROM muc_light_config WHERE room_id = ?">>), + %% This query uses multiple tables + mongoose_rdbms:prepare(muc_light_select_config_by_us, muc_light_rooms, + [luser, lserver], + <<"SELECT version, opt, val " + "FROM muc_light_rooms AS r " + "LEFT OUTER JOIN muc_light_config AS c ON r.id = c.room_id " + "WHERE r.luser = ? AND r.lserver = ?">>), + mongoose_rdbms:prepare(muc_light_insert_config, muc_light_config, + [room_id, opt, val], + <<"INSERT INTO muc_light_config (room_id, opt, val)" + " VALUES(?, ?, ?)">>), + mongoose_rdbms:prepare(muc_light_update_config, muc_light_config, + [val, room_id, opt], + <<"UPDATE muc_light_config SET val = ? " + "WHERE room_id = ? AND opt = ?">>), + mongoose_rdbms:prepare(muc_light_delete_config, muc_light_config, + [room_id], + <<"DELETE FROM muc_light_config WHERE room_id = ?">>), + ok. + +prepare_blocking_queries(_Host) -> + mongoose_rdbms:prepare(muc_light_select_blocking, muc_light_blocking, + [luser, lserver], + <<"SELECT what, who FROM muc_light_blocking " + "WHERE luser = ? AND lserver = ?">>), + mongoose_rdbms:prepare(muc_light_select_blocking_cnt, muc_light_blocking, + [luser, lserver, what, who], + <<"SELECT COUNT(*) FROM muc_light_blocking " + "WHERE luser = ? AND lserver = ? AND " + "what = ? AND who = ?">>), + mongoose_rdbms:prepare(muc_light_select_blocking_cnt2, muc_light_blocking, + [luser, lserver, what, who, what, who], + <<"SELECT COUNT(*) FROM muc_light_blocking " + "WHERE luser = ? AND lserver = ? AND " + "((what = ? AND who = ?) OR (what = ? AND who = ?))">>), + mongoose_rdbms:prepare(muc_light_insert_blocking, muc_light_blocking, + [luser, lserver, what, who], + <<"INSERT INTO muc_light_blocking" + " (luser, lserver, what, who)" + " VALUES (?, ?, ?, ?)">>), + mongoose_rdbms:prepare(muc_light_delete_blocking1, muc_light_blocking, + [luser, lserver, what, who], + <<"DELETE FROM muc_light_blocking " + "WHERE luser = ? AND lserver = ? AND what = ? AND who = ?">>), + mongoose_rdbms:prepare(muc_light_delete_blocking, muc_light_blocking, + [luser, lserver], + <<"DELETE FROM muc_light_blocking" + " WHERE luser = ? AND lserver = ?">>), + ok. + +%% ------------------------ Room SQL functions ------------------------ + +select_room_id(MainHost, RoomU, RoomS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_room_id, [RoomU, RoomS]). + +select_room_id_and_version(MainHost, RoomU, RoomS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_room_id_and_version, [RoomU, RoomS]). + +select_user_rooms(MainHost, LUser, LServer) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_user_rooms, [LUser, LServer]). + +select_user_rooms_count(MainHost, LUser, LServer) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_user_rooms_count, [LUser, LServer]). + +insert_room(MainHost, RoomU, RoomS, Version) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_insert_room, [RoomU, RoomS, Version]). + +update_room_version(MainHost, RoomU, RoomS, Version) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_update_room_version, [Version, RoomU, RoomS]). + +delete_room(MainHost, RoomU, RoomS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_room, [RoomU, RoomS]). + +%% ------------------------ Affiliation SQL functions ------------------------ + +%% Returns affiliations with a version +select_affs_by_us(MainHost, RoomU, RoomS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_affs_by_us, [RoomU, RoomS]). + +%% Returns affiliations without a version +select_affs_by_room_id(MainHost, RoomID) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_affs_by_room_id, [RoomID]). + +insert_aff(MainHost, RoomID, UserU, UserS, Aff) -> + DbAff = aff_atom2db(Aff), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_insert_aff, [RoomID, UserU, UserS, DbAff]). + +update_aff(MainHost, RoomID, UserU, UserS, Aff) -> + DbAff = aff_atom2db(Aff), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_update_aff, [DbAff, RoomID, UserU, UserS]). + +delete_affs(MainHost, RoomID) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_affs, [RoomID]). + +delete_aff(MainHost, RoomID, UserU, UserS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_aff, [RoomID, UserU, UserS]). + +%% ------------------------ Config SQL functions --------------------------- +select_config_by_room_id(MainHost, RoomID) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_config_by_room_id, [RoomID]). + +select_config_by_us(MainHost, RoomU, RoomS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_config_by_us, [RoomU, RoomS]). + +insert_config(MainHost, RoomID, Key, Val) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_insert_config, [RoomID, Key, Val]). + +update_config(MainHost, RoomID, Key, Val) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_update_config, [Val, RoomID, Key]). + +delete_config(MainHost, RoomID) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_config, [RoomID]). + +%% ------------------------ Blocking SQL functions ------------------------- + +select_blocking(MainHost, LUser, LServer) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_blocking, [LUser, LServer]). + +select_blocking_cnt(MainHost, LUser, LServer, [{What, Who}]) -> + DbWhat = what_atom2db(What), + DbWho = jid:to_binary(Who), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_blocking_cnt, + [LUser, LServer, DbWhat, DbWho]); +select_blocking_cnt(MainHost, LUser, LServer, [{What1, Who1}, {What2, Who2}]) -> + DbWhat1 = what_atom2db(What1), + DbWhat2 = what_atom2db(What2), + DbWho1 = jid:to_binary(Who1), + DbWho2 = jid:to_binary(Who2), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_blocking_cnt2, + [LUser, LServer, DbWhat1, DbWho1, DbWhat2, DbWho2]). + +insert_blocking(MainHost, LUser, LServer, What, Who) -> + DbWhat = what_atom2db(What), + DbWho = jid:to_binary(Who), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_insert_blocking, + [LUser, LServer, DbWhat, DbWho]). + +delete_blocking1(MainHost, LUser, LServer, What, Who) -> + DbWhat = what_atom2db(What), + DbWho = jid:to_binary(Who), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_blocking1, + [LUser, LServer, DbWhat, DbWho]). + +delete_blocking(MainHost, UserU, UserS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_blocking, [UserU, UserS]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), AffUsers :: aff_users(), Version :: binary()) -> {ok, FinalRoomUS :: jid:simple_bare_jid()} | {error, exists}. +create_room({<<>>, RoomS} = RoomUS, Config, AffUsers, Version) -> + MainHost = main_host(RoomUS), + create_room_with_random_name(MainHost, RoomS, Config, AffUsers, Version, 10); create_room(RoomUS, Config, AffUsers, Version) -> MainHost = main_host(RoomUS), - {atomic, Res} - = mongoose_rdbms:sql_transaction( - MainHost, fun() -> create_room_transaction(RoomUS, Config, AffUsers, Version) end), - Res. + create_room_with_specified_name(MainHost, RoomUS, Config, AffUsers, Version). + +create_room_with_random_name(_MainHost, RoomS, _Config, _AffUsers, _Version, 0) -> + ?LOG_ERROR(#{what => muc_create_room_with_random_name_failed, + sub_host => RoomS}), + error(create_room_with_random_name_failed); +create_room_with_random_name(MainHost, RoomS, Config, AffUsers, Version, Retries) + when Retries > 0 -> + RoomU = mongoose_bin:gen_from_timestamp(), + RoomUS = {RoomU, RoomS}, + F = fun() -> create_room_transaction(MainHost, RoomUS, Config, AffUsers, Version) end, + case mongoose_rdbms:sql_transaction(MainHost, F) of + {atomic, ok} -> + {ok, RoomUS}; + Other -> + ?LOG_ERROR(#{what => muc_create_room_with_random_name_retry, + candidate_room => RoomU, sub_host => RoomS, reason => Other}), + create_room_with_random_name(MainHost, RoomS, Config, AffUsers, Version, Retries-1) + end. + +create_room_with_specified_name(MainHost, RoomUS, Config, AffUsers, Version) -> + F = fun() -> create_room_transaction(MainHost, RoomUS, Config, AffUsers, Version) end, + case mongoose_rdbms:sql_transaction(MainHost, F) of + {atomic, ok} -> + {ok, RoomUS}; + Other -> + case room_exists(RoomUS) of + true -> + {error, exists}; + false -> %% Some unknown error condition + {RoomU, RoomS} = RoomUS, + ?LOG_ERROR(#{what => muc_create_room_with_specified_name_failed, + room => RoomU, sub_host => RoomS, reason => Other}), + error(create_room_with_specified_name_failed) + end + end. -spec destroy_room(RoomUS :: jid:simple_bare_jid()) -> ok | {error, not_exists | not_empty}. destroy_room(RoomUS) -> MainHost = main_host(RoomUS), - {atomic, Res} - = mongoose_rdbms:sql_transaction(MainHost, fun() -> destroy_room_transaction(RoomUS) end), + F = fun() -> destroy_room_transaction(MainHost, RoomUS) end, + {atomic, Res} = mongoose_rdbms:sql_transaction(MainHost, F), Res. -spec room_exists(RoomUS :: jid:simple_bare_jid()) -> boolean(). room_exists({RoomU, RoomS} = RoomUS) -> MainHost = main_host(RoomUS), - {selected, Res} = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_room_id(RoomU, RoomS)), + {selected, Res} = select_room_id(MainHost, RoomU, RoomS), Res /= []. -spec get_user_rooms(UserUS :: jid:simple_bare_jid(), MUCServer :: jid:lserver() | undefined) -> [RoomUS :: jid:simple_bare_jid()]. get_user_rooms({LUser, LServer}, undefined) -> - SQL = mod_muc_light_db_rdbms_sql:select_user_rooms(LUser, LServer), lists:usort(lists:flatmap( fun(Host) -> - {selected, Rooms} = mongoose_rdbms:sql_query(Host, SQL), + {selected, Rooms} = select_user_rooms(Host, LUser, LServer), Rooms end, ?MYHOSTS)); get_user_rooms({LUser, LServer}, MUCServer) -> MainHost = main_host(MUCServer), - {selected, Rooms} = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_user_rooms(LUser, LServer)), + {selected, Rooms} = select_user_rooms(MainHost, LUser, LServer), Rooms. -spec get_user_rooms_count(UserUS :: jid:simple_bare_jid(), @@ -126,16 +417,14 @@ get_user_rooms({LUser, LServer}, MUCServer) -> non_neg_integer(). get_user_rooms_count({LUser, LServer}, MUCServer) -> MainHost = main_host(MUCServer), - {selected, [{Cnt}]} - = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_user_rooms_count(LUser, LServer)), + {selected, [{Cnt}]} = select_user_rooms_count(MainHost, LUser, LServer), mongoose_rdbms:result_to_integer(Cnt). -spec remove_user(UserUS :: jid:simple_bare_jid(), Version :: binary()) -> mod_muc_light_db:remove_user_return() | {error, term()}. remove_user({_, UserS} = UserUS, Version) -> - {atomic, Res} - = mongoose_rdbms:sql_transaction(UserS, fun() -> remove_user_transaction(UserUS, Version) end), + F = fun() -> remove_user_transaction(UserS, UserUS, Version) end, + {atomic, Res} = mongoose_rdbms:sql_transaction(UserS, F), Res. %% ------------------------ Configuration manipulation ------------------------ @@ -144,10 +433,7 @@ remove_user({_, UserS} = UserUS, Version) -> {ok, mod_muc_light_room_config:kv(), Version :: binary()} | {error, not_exists}. get_config({RoomU, RoomS} = RoomUS) -> MainHost = main_host(RoomUS), - - SQL = mod_muc_light_db_rdbms_sql:select_config(RoomU, RoomS), - {selected, Result} = mongoose_rdbms:sql_query(MainHost, SQL), - + {selected, Result} = select_config_by_us(MainHost, RoomU, RoomS), case Result of [] -> {error, not_exists}; @@ -160,29 +446,6 @@ get_config({RoomU, RoomS} = RoomUS) -> {ok, Config, Version} end. --spec get_config(RoomUS :: jid:simple_bare_jid(), Key :: atom()) -> - {ok, term(), Version :: binary()} | {error, not_exists | invalid_opt}. -get_config({RoomU, RoomS} = RoomUS, Key) -> - MainHost = main_host(RoomUS), - ConfigSchema = mod_muc_light:config_schema(RoomS), - {ok, KeyDB} = mod_muc_light_room_config:schema_reverse_find(Key, ConfigSchema), - - SQL = mod_muc_light_db_rdbms_sql:select_config(RoomU, RoomS, KeyDB), - {selected, Result} = mongoose_rdbms:sql_query(MainHost, SQL), - - case Result of - [] -> - {error, not_exists}; - [{_Version, null, null}] -> - {error, invalid_opt}; - [{Version, _, ValDB}] -> - RawConfig = [{KeyDB, ValDB}], - {ok, [{_, Val}]} = mod_muc_light_room_config:apply_binary_kv( - RawConfig, [], ConfigSchema), - {ok, Val, Version} - end. - - -spec set_config(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), Version :: binary()) -> {ok, PrevVersion :: binary()} | {error, not_exists}. @@ -205,18 +468,16 @@ set_config(RoomJID, Key, Val, Version) -> [blocking_item()]. get_blocking({LUser, LServer}, MUCServer) -> MainHost = main_host(MUCServer), - SQL = mod_muc_light_db_rdbms_sql:select_blocking(LUser, LServer), - {selected, WhatWhos} = mongoose_rdbms:sql_query(MainHost, SQL), - [ {what_db2atom(What), deny, jid:to_lus(jid:from_binary(Who))} || {What, Who} <- WhatWhos ]. + {selected, WhatWhos} = select_blocking(MainHost, LUser, LServer), + decode_blocking(WhatWhos). -spec get_blocking(UserUS :: jid:simple_bare_jid(), MUCServer :: jid:lserver(), - WhatWhos :: [{blocking_who(), jid:simple_bare_jid()}]) -> + WhatWhos :: [{blocking_what(), jid:simple_bare_jid()}]) -> blocking_action(). get_blocking({LUser, LServer}, MUCServer, WhatWhos) -> MainHost = main_host(MUCServer), - SQL = mod_muc_light_db_rdbms_sql:select_blocking_cnt(LUser, LServer, WhatWhos), - {selected, [{Count}]} = mongoose_rdbms:sql_query(MainHost, SQL), + {selected, [{Count}]} = select_blocking_cnt(MainHost, LUser, LServer, WhatWhos), case mongoose_rdbms:result_to_integer(Count) of 0 -> allow; _ -> deny @@ -225,18 +486,20 @@ get_blocking({LUser, LServer}, MUCServer, WhatWhos) -> -spec set_blocking(UserUS :: jid:simple_bare_jid(), MUCServer :: jid:lserver(), BlockingItems :: [blocking_item()]) -> ok. -set_blocking(_UserUS, _MUCServer, []) -> +set_blocking(UserUS, MUCServer, BlockingItems) -> + MainHost = main_host(MUCServer), + set_blocking_loop(MainHost, UserUS, MUCServer, BlockingItems). + +set_blocking_loop(_MainHost, _UserUS, _MUCServer, []) -> ok; -set_blocking({LUser, LServer} = UserUS, MUCServer, [{What, deny, Who} | RBlockingItems]) -> - {updated, _} = - mongoose_rdbms:sql_query( - main_host(MUCServer), mod_muc_light_db_rdbms_sql:insert_blocking(LUser, LServer, What, Who)), - set_blocking(UserUS, MUCServer, RBlockingItems); -set_blocking({LUser, LServer} = UserUS, MUCServer, [{What, allow, Who} | RBlockingItems]) -> - {updated, _} = - mongoose_rdbms:sql_query( - main_host(MUCServer), mod_muc_light_db_rdbms_sql:delete_blocking(LUser, LServer, What, Who)), - set_blocking(UserUS, MUCServer, RBlockingItems). +set_blocking_loop(MainHost, {LUser, LServer} = UserUS, MUCServer, + [{What, deny, Who} | RBlockingItems]) -> + {updated, _} = insert_blocking(MainHost, LUser, LServer, What, Who), + set_blocking_loop(MainHost, UserUS, MUCServer, RBlockingItems); +set_blocking_loop(MainHost, {LUser, LServer} = UserUS, MUCServer, + [{What, allow, Who} | RBlockingItems]) -> + {updated, _} = delete_blocking1(MainHost, LUser, LServer, What, Who), + set_blocking_loop(MainHost, UserUS, MUCServer, RBlockingItems). %% ------------------------ Affiliations manipulation ------------------------ @@ -244,14 +507,14 @@ set_blocking({LUser, LServer} = UserUS, MUCServer, [{What, allow, Who} | RBlocki {ok, aff_users(), Version :: binary()} | {error, not_exists}. get_aff_users({RoomU, RoomS} = RoomUS) -> MainHost = main_host(RoomUS), - case mongoose_rdbms:sql_query(MainHost, mod_muc_light_db_rdbms_sql:select_affs(RoomU, RoomS)) of + case select_affs_by_us(MainHost, RoomU, RoomS) of {selected, []} -> {error, not_exists}; {selected, [{Version, null, null, null}]} -> {ok, [], Version}; {selected, [{Version, _, _, _} | _] = Res} -> - AffUsers = [{{UserU, UserS}, aff_db2atom(Aff)} || {_, UserU, UserS, Aff} <- Res], - {ok, lists:sort(AffUsers), Version} + AffUsers = decode_affs_with_versions(Res), + {ok, AffUsers, Version} end. -spec modify_aff_users(RoomUS :: jid:simple_bare_jid(), @@ -261,10 +524,9 @@ get_aff_users({RoomU, RoomS} = RoomUS) -> mod_muc_light_db:modify_aff_users_return(). modify_aff_users(RoomUS, AffUsersChanges, ExternalCheck, Version) -> MainHost = main_host(RoomUS), - {atomic, Res} - = mongoose_rdbms:sql_transaction( - MainHost, fun() -> modify_aff_users_transaction( - RoomUS, AffUsersChanges, ExternalCheck, Version) end), + F = fun() -> modify_aff_users_transaction(MainHost, RoomUS, AffUsersChanges, + ExternalCheck, Version) end, + {atomic, Res} = mongoose_rdbms:sql_transaction(MainHost, F), Res. %% ------------------------ Misc ------------------------ @@ -274,37 +536,48 @@ modify_aff_users(RoomUS, AffUsersChanges, ExternalCheck, Version) -> | {error, not_exists}. get_info({RoomU, RoomS} = RoomUS) -> MainHost = main_host(RoomUS), - case mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_room_id_and_version(RoomU, RoomS)) of - {selected, [{RoomID, Version}]} -> - {selected, AffUsersDB} = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_affs(RoomID)), - AffUsers = [{{UserU, UserS}, aff_db2atom(Aff)} || {UserU, UserS, Aff} <- AffUsersDB], - - {selected, ConfigDB} = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_config(RoomID)), + case select_room_id_and_version(MainHost, RoomU, RoomS) of + {selected, [{DbRoomID, Version}]} -> + RoomID = mongoose_rdbms:result_to_integer(DbRoomID), + {selected, AffUsersDB} = select_affs_by_room_id(MainHost, RoomID), + AffUsers = decode_affs(AffUsersDB), + {selected, ConfigDB} = select_config_by_room_id(MainHost, RoomID), {ok, Config} = mod_muc_light_room_config:apply_binary_kv( ConfigDB, [], mod_muc_light:config_schema(RoomS)), - {ok, Config, lists:sort(AffUsers), Version}; + {ok, Config, AffUsers, Version}; {selected, []} -> {error, not_exists} end. %% ------------------------ Conversions ------------------------ +decode_affs(AffUsersDB) -> + US2Aff = [{{UserU, UserS}, aff_db2atom(Aff)} + || {UserU, UserS, Aff} <- AffUsersDB], + lists:sort(US2Aff). + +decode_affs_with_versions(AffUsersDB) -> + US2Aff = [{{UserU, UserS}, aff_db2atom(Aff)} + || {_Version, UserU, UserS, Aff} <- AffUsersDB], + lists:sort(US2Aff). + +decode_blocking(WhatWhos) -> + [ {what_db2atom(What), deny, jid:to_lus(jid:from_binary(Who))} + || {What, Who} <- WhatWhos ]. + -spec what_db2atom(binary() | pos_integer()) -> blocking_what(). what_db2atom(1) -> room; what_db2atom(2) -> user; what_db2atom(Bin) -> what_db2atom(mongoose_rdbms:result_to_integer(Bin)). --spec what_atom2db(blocking_what()) -> string(). -what_atom2db(room) -> "1"; -what_atom2db(user) -> "2". +-spec what_atom2db(blocking_what()) -> non_neg_integer(). +what_atom2db(room) -> 1; +what_atom2db(user) -> 2. --spec aff_atom2db(aff()) -> string(). -aff_atom2db(owner) -> "1"; -aff_atom2db(member) -> "2". +-spec aff_atom2db(aff()) -> non_neg_integer(). +aff_atom2db(owner) -> 1; +aff_atom2db(member) -> 2. -spec aff_db2atom(binary() | pos_integer()) -> aff(). aff_db2atom(1) -> owner; @@ -315,15 +588,17 @@ aff_db2atom(Bin) -> aff_db2atom(mongoose_rdbms:result_to_integer(Bin)). %% API for tests %%==================================================================== +force_clear_statements() -> + [muc_light_config_delete_all, + muc_light_occupants_delete_all, + muc_light_rooms_delete_all, + muc_light_blocking_delete_all]. + -spec force_clear() -> ok. force_clear() -> - lists:foreach( - fun(Host) -> - mongoose_rdbms:sql_query(Host, ["DELETE FROM muc_light_config"]), - mongoose_rdbms:sql_query(Host, ["DELETE FROM muc_light_occupants"]), - mongoose_rdbms:sql_query(Host, ["DELETE FROM muc_light_rooms"]), - mongoose_rdbms:sql_query(Host, ["DELETE FROM muc_light_blocking"]) - end, ?MYHOSTS). + [mongoose_rdbms:execute_successfully(Host, Statement, []) + || Host <- ?MYHOSTS, Statement <- force_clear_statements()], + ok. %%==================================================================== %% Internal functions @@ -332,83 +607,51 @@ force_clear() -> %% ------------------------ General room management ------------------------ %% Expects config to have unique fields! --spec create_room_transaction(RoomUS :: jid:simple_bare_jid(), +-spec create_room_transaction(MainHost :: jid:lserver(), + RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), AffUsers :: aff_users(), - Version :: binary()) -> - {ok, FinalRoomUS :: jid:simple_bare_jid()} | {error, exists}. -create_room_transaction({NodeCandidate, RoomS}, Config, AffUsers, Version) -> - RoomU = case NodeCandidate of - <<>> -> mongoose_bin:gen_from_timestamp(); - _ -> NodeCandidate - end, - case catch mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:insert_room(RoomU, RoomS, Version)) of - {aborted, Reason} -> - %% At this point the transaction is broken because of failed INSERT query - mongoose_rdbms:sql_query_t("ROLLBACK;"), - mongoose_rdbms:sql_query_t(rdbms_queries:begin_trans()), - - case {mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:select_room_id(RoomU, RoomS)), NodeCandidate} of - {{selected, []}, _} -> - throw({aborted, Reason}); - {{selected, [_]}, <<>>} -> - create_room_transaction({<<>>, RoomS}, Config, AffUsers, Version); - {{selected, [_]}, _} -> - {error, exists} - end; - {updated, _} -> - {selected, [{RoomID} | Rest] = AllIds} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:select_room_id(RoomU, RoomS)), - case Rest of - [] -> - ok; - _ -> - Details = <<"Many IDs returned for PK select query, most probably MSSQL deadlock">>, - ?LOG_ERROR(#{what => muc_many_ids_for_pk_select, text => Details, - room => RoomU, sub_host => RoomS, all_room_ids => AllIds}), - throw({aborted, Details}) - end, - lists:foreach( - fun({{UserU, UserS}, Aff}) -> - Query = mod_muc_light_db_rdbms_sql:insert_aff(RoomID, UserU, UserS, Aff), - {updated, _} = mongoose_rdbms:sql_query_t(Query) - end, AffUsers), - ConfigFields = mod_muc_light_room_config:to_binary_kv( - Config, mod_muc_light:config_schema(RoomS)), - lists:foreach( - fun({Key, Val}) -> - Query = mod_muc_light_db_rdbms_sql:insert_config(RoomID, Key, Val), - {updated, _} = mongoose_rdbms:sql_query_t(Query) - end, ConfigFields), - {ok, {RoomU, RoomS}} - end. + Version :: binary()) -> ok. +create_room_transaction(MainHost, {RoomU, RoomS}, Config, AffUsers, Version) -> + insert_room(MainHost, RoomU, RoomS, Version), + RoomID = mongoose_rdbms:selected_to_integer(select_room_id(MainHost, RoomU, RoomS)), + Schema = mod_muc_light:config_schema(RoomS), + ConfigFields = mod_muc_light_room_config:to_binary_kv(Config, Schema), + [insert_aff_tuple(MainHost, RoomID, AffUser) || AffUser <- AffUsers], + [insert_config_kv(MainHost, RoomID, KV) || KV <- ConfigFields], + ok. --spec destroy_room_transaction(RoomUS :: jid:simple_bare_jid()) -> ok | {error, not_exists}. -destroy_room_transaction({RoomU, RoomS}) -> - case mongoose_rdbms:sql_query_t(mod_muc_light_db_rdbms_sql:select_room_id(RoomU, RoomS)) of - {selected, [{RoomID}]} -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:delete_affs(RoomID)), - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:delete_config(RoomID)), - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:delete_room(RoomU, RoomS)), +insert_aff_tuple(MainHost, RoomID, {{UserU, UserS}, Aff}) -> + insert_aff(MainHost, RoomID, UserU, UserS, Aff). + +insert_config_kv(MainHost, RoomID, {Key, Val}) -> + insert_config(MainHost, RoomID, Key, Val). + +-spec destroy_room_transaction(MainHost :: jid:lserver(), + RoomUS :: jid:simple_bare_jid()) -> + ok | {error, not_exists}. +destroy_room_transaction(MainHost, {RoomU, RoomS}) -> + case select_room_id(MainHost, RoomU, RoomS) of + {selected, [{DbRoomID}]} -> + RoomID = mongoose_rdbms:result_to_integer(DbRoomID), + {updated, _} = delete_affs(MainHost, RoomID), + {updated, _} = delete_config(MainHost, RoomID), + {updated, _} = delete_room(MainHost, RoomU, RoomS), ok; {selected, []} -> {error, not_exists} end. --spec remove_user_transaction(UserUS :: jid:simple_bare_jid(), Version :: binary()) -> +-spec remove_user_transaction(MainHost :: jid:lserver(), + UserUS :: jid:simple_bare_jid(), + Version :: binary()) -> mod_muc_light_db:remove_user_return(). -remove_user_transaction({UserU, UserS} = UserUS, Version) -> +remove_user_transaction(MainHost, {UserU, UserS} = UserUS, Version) -> Rooms = get_user_rooms(UserUS, undefined), - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:delete_blocking(UserU, UserS)), + {updated, _} = delete_blocking(MainHost, UserU, UserS), lists:map( fun(RoomUS) -> - {RoomUS, modify_aff_users_transaction( + {RoomUS, modify_aff_users_transaction(MainHost, RoomUS, [{UserUS, none}], fun(_, _) -> ok end, Version)} end, Rooms). @@ -420,16 +663,13 @@ remove_user_transaction({UserU, UserS} = UserUS, Version) -> {ok, PrevVersion :: binary()} | {error, not_exists}. set_config_transaction({RoomU, RoomS} = RoomUS, ConfigChanges, Version) -> MainHost = main_host(RoomUS), - case mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:select_room_id_and_version(RoomU, RoomS)) of - {selected, [{RoomID, PrevVersion}]} -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:update_room_version(RoomU, RoomS, Version)), + case select_room_id_and_version(MainHost, RoomU, RoomS) of + {selected, [{DbRoomID, PrevVersion}]} -> + RoomID = mongoose_rdbms:result_to_integer(DbRoomID), + {updated, _} = update_room_version(MainHost, RoomU, RoomS, Version), lists:foreach( fun({Key, Val}) -> - {updated, _} - = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:update_config(RoomID, Key, Val)) + {updated, _} = update_config(MainHost, RoomID, Key, Val) end, mod_muc_light_room_config:to_binary_kv( ConfigChanges, mod_muc_light:config_schema(RoomS))), {ok, PrevVersion}; @@ -441,39 +681,42 @@ set_config_transaction({RoomU, RoomS} = RoomUS, ConfigChanges, Version) -> %% ------------------------ Affiliations manipulation ------------------------ --spec modify_aff_users_transaction(RoomUS :: jid:simple_bare_jid(), +-spec modify_aff_users_transaction(MainHost :: jid:lserver(), + RoomUS :: jid:simple_bare_jid(), AffUsersChanges :: aff_users(), CheckFun :: external_check_fun(), Version :: binary()) -> mod_muc_light_db:modify_aff_users_return(). -modify_aff_users_transaction({RoomU, RoomS} = RoomUS, AffUsersChanges, CheckFun, Version) -> - case mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:select_room_id_and_version(RoomU, RoomS)) of - {selected, [{RoomID, PrevVersion}]} -> - modify_aff_users_transaction( +modify_aff_users_transaction(MainHost, {RoomU, RoomS} = RoomUS, + AffUsersChanges, CheckFun, Version) -> + case select_room_id_and_version(MainHost, RoomU, RoomS) of + {selected, [{DbRoomID, PrevVersion}]} -> + RoomID = mongoose_rdbms:result_to_integer(DbRoomID), + modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, CheckFun, PrevVersion, Version); {selected, []} -> {error, not_exists} end. --spec modify_aff_users_transaction(RoomUS :: jid:simple_bare_jid(), - RoomID :: binary(), +-spec modify_aff_users_transaction(MainHost :: jid:lserver(), + RoomUS :: jid:simple_bare_jid(), + RoomID :: room_id(), AffUsersChanges :: aff_users(), CheckFun :: external_check_fun(), PrevVersion :: binary(), Version :: binary()) -> mod_muc_light_db:modify_aff_users_return(). -modify_aff_users_transaction(RoomUS, RoomID, AffUsersChanges, CheckFun, PrevVersion, Version) -> - {selected, AffUsersDB} - = mongoose_rdbms:sql_query_t(mod_muc_light_db_rdbms_sql:select_affs(RoomID)), - AffUsers = lists:sort( - [{{UserU, UserS}, aff_db2atom(Aff)} || {UserU, UserS, Aff} <- AffUsersDB]), +modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, + CheckFun, PrevVersion, Version) -> + {selected, AffUsersDB} = select_affs_by_room_id(MainHost, RoomID), + AffUsers = decode_affs(AffUsersDB), case mod_muc_light_utils:change_aff_users(AffUsers, AffUsersChanges) of {ok, NewAffUsers, AffUsersChanged, JoiningUsers, _LeavingUsers} -> case CheckFun(RoomUS, NewAffUsers) of ok -> - apply_aff_users_transaction(RoomID, AffUsersChanged, JoiningUsers), - update_room_version_transaction(RoomUS, Version), + apply_aff_users_transaction(MainHost, RoomID, AffUsersChanged, JoiningUsers), + {RoomU, RoomS} = RoomUS, + {updated, _} = update_room_version(MainHost, RoomU, RoomS, Version), {ok, AffUsers, NewAffUsers, AffUsersChanged, PrevVersion}; Error -> Error @@ -482,33 +725,23 @@ modify_aff_users_transaction(RoomUS, RoomID, AffUsersChanges, CheckFun, PrevVers Error end. --spec apply_aff_users_transaction(RoomID :: binary(), +-spec apply_aff_users_transaction(MainHost :: jid:lserver(), + RoomID :: room_id(), AffUsersChanges :: aff_users(), JoiningUsers :: [jid:simple_bare_jid()]) -> ok. -apply_aff_users_transaction(RoomID, AffUsersChanged, JoiningUsers) -> +apply_aff_users_transaction(MainHost, RoomID, AffUsersChanged, JoiningUsers) -> lists:foreach( fun({{UserU, UserS}, none}) -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:delete_aff(RoomID, UserU, UserS)); + {updated, _} = delete_aff(MainHost, RoomID, UserU, UserS); ({{UserU, UserS} = UserUS, Aff}) -> case lists:member(UserUS, JoiningUsers) of true -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:insert_aff( - RoomID, UserU, UserS, Aff)); + {updated, _} = insert_aff(MainHost, RoomID, UserU, UserS, Aff); false -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:update_aff( - RoomID, UserU, UserS, Aff)) + {updated, _} = update_aff(MainHost, RoomID, UserU, UserS, Aff) end end, AffUsersChanged). --spec update_room_version_transaction(RoomUS :: jid:simple_bare_jid(), Version :: binary()) -> - {updated, integer()}. -update_room_version_transaction({RoomU, RoomS}, Version) -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:update_room_version(RoomU, RoomS, Version)). - %% ------------------------ Common ------------------------ -spec main_host(JIDOrServer :: jid:simple_bare_jid() | binary()) -> jid:lserver(). diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl deleted file mode 100644 index 7c8cb447a3..0000000000 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ /dev/null @@ -1,207 +0,0 @@ -%%%---------------------------------------------------------------------- -%%% File : mod_muc_light_db_rdbms_sql.erl -%%% Author : Piotr Nosek -%%% Purpose : RDBMS backend queries for mod_muc_light -%%% Created : 29 Nov 2016 by Piotr Nosek -%%% -%%% This program is free software; you can redistribute it and/or -%%% modify it under the terms of the GNU General Public License as -%%% published by the Free Software Foundation; either version 2 of the -%%% License, or (at your option) any later version. -%%% -%%% This program is distributed in the hope that it will be useful, -%%% but WITHOUT ANY WARRANTY; without even the implied warranty of -%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -%%% General Public License for more details. -%%% -%%% You should have received a copy of the GNU General Public License -%%% along with this program; if not, write to the Free Software -%%% Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -%%% -%%%---------------------------------------------------------------------- - --module(mod_muc_light_db_rdbms_sql). --author('piotr.nosek@erlang-solutions.com'). - --include("mod_muc_light.hrl"). - --export([select_room_id/2, select_room_id_and_version/2, - select_user_rooms/2, select_user_rooms_count/2, - insert_room/3, update_room_version/3, delete_room/2]). --export([select_affs/2, select_affs/1, insert_aff/4, update_aff/4, delete_affs/1, delete_aff/3]). --export([select_config/1, select_config/2, select_config/3, insert_config/3, update_config/3, - delete_config/1]). --export([select_blocking/2, select_blocking_cnt/3, insert_blocking/4, - delete_blocking/4, delete_blocking/2]). - --define(ESC(T), mongoose_rdbms:use_escaped_string(mongoose_rdbms:escape_string(T))). - -%%==================================================================== -%% General room queries -%%==================================================================== - --spec select_room_id(RoomU :: jid:luser(), RoomS :: jid:lserver()) -> iolist(). -select_room_id(RoomU, RoomS) -> - ["SELECT id FROM muc_light_rooms WHERE luser = ", ?ESC(RoomU), - " AND lserver = ", ?ESC(RoomS)]. - --spec select_room_id_and_version( - RoomU :: jid:luser(), RoomS :: jid:lserver()) -> iolist(). -select_room_id_and_version(RoomU, RoomS) -> - ["SELECT id, version FROM muc_light_rooms WHERE luser = ", ?ESC(RoomU), - " AND lserver = ", ?ESC(RoomS)]. - --spec select_user_rooms(LUser :: jid:luser(), LServer :: jid:lserver()) -> iolist(). -select_user_rooms(LUser, LServer) -> - select_user_rooms(LUser, LServer, "r.luser, r.lserver"). - --spec select_user_rooms_count(LUser :: jid:luser(), LServer :: jid:lserver()) -> iolist(). -select_user_rooms_count(LUser, LServer) -> - select_user_rooms(LUser, LServer, "COUNT(*)"). - --spec select_user_rooms(LUser :: jid:luser(), - LServer :: jid:lserver(), - ReturnStatement :: iodata()) -> iolist(). -select_user_rooms(LUser, LServer, ReturnStatement) -> - ["SELECT ", ReturnStatement, - " FROM muc_light_occupants AS o INNER JOIN muc_light_rooms AS r ON o.room_id = r.id" - " WHERE o.luser = ", ?ESC(LUser), " AND o.lserver = ", ?ESC(LServer)]. - --spec insert_room( - RoomU :: jid:luser(), RoomS :: jid:lserver(), Version :: binary()) -> iolist(). -insert_room(RoomU, RoomS, Version) -> - ["INSERT INTO muc_light_rooms (luser, lserver, version)" - " VALUES (", ?ESC(RoomU), ", ", ?ESC(RoomS), ", ", ?ESC(Version), ")"]. - --spec update_room_version( - RoomU :: jid:luser(), RoomS :: jid:lserver(), Version :: binary()) -> iolist(). -update_room_version(RoomU, RoomS, Version) -> - ["UPDATE muc_light_rooms SET version = ", ?ESC(Version), - " WHERE luser = ", ?ESC(RoomU), " AND lserver = ", ?ESC(RoomS)]. - --spec delete_room(RoomU :: jid:luser(), RoomS :: jid:lserver()) -> iolist(). -delete_room(RoomU, RoomS) -> - ["DELETE FROM muc_light_rooms" - " WHERE luser = ", ?ESC(RoomU), " AND lserver = ", ?ESC(RoomS)]. - -%%==================================================================== -%% Affiliations -%%==================================================================== - --spec select_affs(RoomU :: jid:luser(), RoomS :: jid:lserver()) -> iolist(). -select_affs(RoomU, RoomS) -> - ["SELECT version, o.luser, o.lserver, aff" - " FROM muc_light_rooms AS r LEFT OUTER JOIN muc_light_occupants AS o ON r.id = o.room_id" - " WHERE r.luser = ", ?ESC(RoomU), " AND r.lserver = ", ?ESC(RoomS)]. - --spec select_affs(RoomID :: integer() | binary()) -> iolist(). -select_affs(RoomID) -> - ["SELECT luser, lserver, aff FROM muc_light_occupants WHERE room_id = ", bin(RoomID)]. - --spec insert_aff(RoomID :: integer() | binary(), UserU :: jid:luser(), - UserS :: jid:lserver(), Aff :: aff()) -> iolist(). -insert_aff(RoomID, UserU, UserS, Aff) -> - ["INSERT INTO muc_light_occupants (room_id, luser, lserver, aff)" - " VALUES(", bin(RoomID), ", ", ?ESC(UserU), ", ", ?ESC(UserS), ", ", - mod_muc_light_db_rdbms:aff_atom2db(Aff), ")"]. - --spec update_aff(RoomID :: integer() | binary(), UserU :: jid:luser(), - UserS :: jid:lserver(), Aff :: aff()) -> iolist(). -update_aff(RoomID, UserU, UserS, Aff) -> - ["UPDATE muc_light_occupants SET aff = ", mod_muc_light_db_rdbms:aff_atom2db(Aff), - " WHERE room_id = ", bin(RoomID), " AND luser = ", ?ESC(UserU), - " AND lserver = ", ?ESC(UserS)]. - --spec delete_affs(RoomID :: integer() | binary()) -> iolist(). -delete_affs(RoomID) -> - ["DELETE FROM muc_light_occupants WHERE room_id = ", bin(RoomID)]. - --spec delete_aff(RoomID :: integer() | binary(), UserU :: jid:luser(), - UserS :: jid:lserver()) -> - iolist(). -delete_aff(RoomID, UserU, UserS) -> - ["DELETE FROM muc_light_occupants WHERE room_id = ", bin(RoomID), - " AND luser = ", ?ESC(UserU), - " AND lserver = ", ?ESC(UserS)]. - -%%==================================================================== -%% Config -%%==================================================================== - --spec select_config(RoomID :: integer() | binary()) -> iolist(). -select_config(RoomID) -> - ["SELECT opt, val FROM muc_light_config WHERE room_id = ", bin(RoomID)]. - --spec select_config(RoomU :: jid:luser(), RoomS :: jid:lserver()) -> iolist(). -select_config(RoomU, RoomS) -> - ["SELECT version, opt, val", - " FROM muc_light_rooms AS r LEFT OUTER JOIN muc_light_config AS c ON r.id = c.room_id" - " WHERE r.luser = ", ?ESC(RoomU), " AND r.lserver = ", ?ESC(RoomS)]. - --spec select_config(RoomU :: jid:luser(), RoomS :: jid:lserver(), Key :: binary()) -> - iolist(). -select_config(RoomU, RoomS, Key) -> - [ select_config(RoomU, RoomS), " AND key = '", Key, "'" ]. - --spec insert_config(RoomID :: integer() | binary(), Key :: binary(), Val :: binary()) -> iolist(). -insert_config(RoomID, Key, Val) -> - ["INSERT INTO muc_light_config (room_id, opt, val)" - " VALUES(", bin(RoomID), ", ", ?ESC(Key), ", ", ?ESC(Val), ")"]. - --spec update_config(RoomID :: integer() | binary(), Key :: binary(), Val :: binary()) -> iolist(). -update_config(RoomID, Key, Val) -> - ["UPDATE muc_light_config SET val = ", ?ESC(Val), - " WHERE room_id = ", bin(RoomID), " AND opt = ", ?ESC(Key)]. - --spec delete_config(RoomID :: integer() | binary()) -> iolist(). -delete_config(RoomID) -> - ["DELETE FROM muc_light_config WHERE room_id = ", bin(RoomID)]. - -%%==================================================================== -%% Blocking -%%==================================================================== - --spec select_blocking(LUser :: jid:luser(), LServer :: jid:lserver()) -> iolist(). -select_blocking(LUser, LServer) -> - ["SELECT what, who FROM muc_light_blocking WHERE luser = ", ?ESC(LUser), - " AND lserver = ", ?ESC(LServer)]. - --spec select_blocking_cnt(LUser :: jid:luser(), LServer :: jid:lserver(), - WhatWhos :: [{blocking_who(), jid:simple_bare_jid()}]) -> iolist(). -select_blocking_cnt(LUser, LServer, WhatWhos) -> - [ _ | WhatWhosWhere ] = lists:flatmap( - fun({What, Who}) -> - [" OR ", "(what = ", mod_muc_light_db_rdbms:what_atom2db(What), - " AND who = ", ?ESC(jid:to_binary(Who)), ")"] end, - WhatWhos), - ["SELECT COUNT(*) FROM muc_light_blocking WHERE luser = ", ?ESC(LUser), - " AND lserver = ", ?ESC(LServer), - " AND (", WhatWhosWhere, ")"]. - --spec insert_blocking(LUser :: jid:luser(), LServer :: jid:lserver(), - What :: blocking_what(), Who :: blocking_who()) -> iolist(). -insert_blocking(LUser, LServer, What, Who) -> - ["INSERT INTO muc_light_blocking (luser, lserver, what, who)" - " VALUES (", ?ESC(LUser), ", ", ?ESC(LServer), ", ", - mod_muc_light_db_rdbms:what_atom2db(What), ", ", ?ESC(jid:to_binary(Who)), ")"]. - --spec delete_blocking(LUser :: jid:luser(), LServer :: jid:lserver(), - What :: blocking_what(), Who :: blocking_who()) -> iolist(). -delete_blocking(LUser, LServer, What, Who) -> - ["DELETE FROM muc_light_blocking WHERE luser = ", ?ESC(LUser), - " AND lserver = ", ?ESC(LServer), - " AND what = ", mod_muc_light_db_rdbms:what_atom2db(What), - " AND who = ", ?ESC(jid:to_binary(Who))]. - --spec delete_blocking(UserU :: jid:luser(), UserS :: jid:lserver()) -> iolist(). -delete_blocking(UserU, UserS) -> - ["DELETE FROM muc_light_blocking" - " WHERE luser = ", ?ESC(UserU), " AND lserver = ", ?ESC(UserS)]. - -%%==================================================================== -%% Helpers -%%==================================================================== - --spec bin(integer() | binary()) -> binary(). -bin(Int) when is_integer(Int) -> integer_to_binary(Int); -bin(Bin) when is_binary(Bin) -> Bin. diff --git a/src/offline/mod_offline_rdbms.erl b/src/offline/mod_offline_rdbms.erl index 9a3019ceb5..5d38a0f587 100644 --- a/src/offline/mod_offline_rdbms.erl +++ b/src/offline/mod_offline_rdbms.erl @@ -34,6 +34,8 @@ remove_old_messages/2, remove_user/2]). +-import(mongoose_rdbms, [prepare/4, execute_successfully/3]). + -include("mongoose.hrl"). -include("jlib.hrl"). -include("mod_offline.hrl"). @@ -41,15 +43,81 @@ -define(OFFLINE_TABLE_LOCK_THRESHOLD, 1000). init(_Host, _Opts) -> + prepare_queries(), ok. +prepare_queries() -> + prepare(offline_insert, offline_message, + [username, server, timestamp, expire, + from_jid, packet, permanent_fields], + <<"INSERT INTO offline_message " + "(username, server, timestamp, expire," + " from_jid, packet, permanent_fields) " + "VALUES (?, ?, ?, ?, ?, ?, ?)">>), + {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(), + prepare(offline_count_limit, offline_message, + rdbms_queries:add_limit_arg(limit, [server, username]), + <<"SELECT ", LimitMSSQL/binary, + " count(*) FROM offline_message " + "WHERE server = ? AND username = ? ", LimitSQL/binary>>), + prepare(offline_select, offline_message, + [server, username, expire], + <<"SELECT timestamp, from_jid, packet, permanent_fields " + "FROM offline_message " + "WHERE server = ? AND username = ? AND " + "(expire IS null OR expire > ?) " + "ORDER BY timestamp">>), + prepare(offline_delete, offline_message, + [server, username], + <<"DELETE FROM offline_message " + "WHERE server = ? AND username = ?">>), + prepare(offline_delete_old, offline_message, + [timestamp], + <<"DELETE FROM offline_message WHERE timestamp < ?">>), + prepare(offline_delete_expired, offline_message, + [expire], + <<"DELETE FROM offline_message " + "WHERE expire IS NOT null AND expire < ?">>). + +execute_count_offline_messages(LUser, LServer, Limit) -> + Args = rdbms_queries:add_limit_arg(Limit, [LServer, LUser]), + execute_successfully(LServer, offline_count_limit, Args). + +execute_fetch_offline_messages(LServer, LUser, ExtTimeStamp) -> + execute_successfully(LServer, offline_select, [LServer, LUser, ExtTimeStamp]). + +execute_remove_expired_offline_messages(LServer, ExtTimeStamp) -> + execute_successfully(LServer, offline_delete_expired, [ExtTimeStamp]). + +execute_remove_old_offline_messages(LServer, ExtTimeStamp) -> + execute_successfully(LServer, offline_delete_old, [ExtTimeStamp]). + +execute_offline_delete(LServer, LUser) -> + execute_successfully(LServer, offline_delete, [LServer, LUser]). + +%% Transactions + +pop_offline_messages(LServer, LUser, ExtTimeStamp) -> + F = fun() -> + Res = execute_fetch_offline_messages(LServer, LUser, ExtTimeStamp), + execute_offline_delete(LServer, LUser), + Res + end, + mongoose_rdbms:sql_transaction(LServer, F). + +push_offline_messages(LServer, Rows) -> + F = fun() -> + [execute_successfully(LServer, offline_insert, Row) + || Row <- Rows], ok + end, + mongoose_rdbms:sql_transaction(LServer, F). + +%% API functions + pop_messages(#jid{} = To) -> US = {LUser, LServer} = jid:to_lus(To), - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - TimeStamp = erlang:system_time(microsecond), - STimeStamp = encode_timestamp(TimeStamp), - case rdbms_queries:pop_offline_messages(LServer, SUser, SServer, STimeStamp) of + ExtTimeStamp = os:system_time(microsecond), + case pop_offline_messages(LServer, LUser, ExtTimeStamp) of {atomic, {selected, Rows}} -> {ok, rows_to_records(US, To, Rows)}; {aborted, Reason} -> @@ -58,123 +126,81 @@ pop_messages(#jid{} = To) -> {error, Reason} end. +%% Fetch messages for GDPR fetch_messages(#jid{} = To) -> US = {LUser, LServer} = jid:to_lus(To), - TimeStamp = erlang:system_time(microsecond), - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - STimeStamp = encode_timestamp(TimeStamp), - case rdbms_queries:fetch_offline_messages(LServer, SUser, SServer, STimeStamp) of - {selected, Rows} -> - {ok, rows_to_records(US, To, Rows)}; - {error, Reason} -> - {error, Reason} - end. - -rows_to_records(US, To, Rows) -> - [row_to_record(US, To, Row) || Row <- Rows]. - -row_to_record(US, To, {STimeStamp, SFrom, SPacket, SPermanentFields}) -> - {ok, Packet} = exml:parse(SPacket), - TimeStamp = mongoose_rdbms:result_to_integer(STimeStamp), - From = jid:from_binary(SFrom), - PermanentFields = extract_permanent_fields(SPermanentFields), - #offline_msg{us = US, - timestamp = TimeStamp, - expire = never, - from = From, - to = To, - packet = Packet, - permanent_fields = PermanentFields}. - -extract_permanent_fields(null) -> - []; %% This is needed in transition period when upgrading to MongooseIM above 3.5.0 -extract_permanent_fields(Term) -> - Unescaped = mongoose_rdbms:unescape_binary(global, Term), - binary_to_term(Unescaped). + ExtTimeStamp = os:system_time(microsecond), + {selected, Rows} = execute_fetch_offline_messages(LServer, LUser, ExtTimeStamp), + {ok, rows_to_records(US, To, Rows)}. write_messages(LUser, LServer, Msgs) -> - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - write_all_messages_t(LServer, SUser, SServer, Msgs). - -count_offline_messages(LUser, LServer, MaxArchivedMsgs) -> - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - count_offline_messages(LUser, LServer, SUser, SServer, MaxArchivedMsgs + 1). - -write_all_messages_t(LServer, SUser, SServer, Msgs) -> - Rows = [record_to_row(SUser, SServer, Msg) || Msg <- Msgs], - case rdbms_queries:push_offline_messages(LServer, Rows) of - {updated, _} -> + Rows = [record_to_row(LUser, LServer, Msg) || Msg <- Msgs], + case push_offline_messages(LServer, Rows) of + {atomic, ok} -> ok; - {aborted, Reason} -> - {error, {aborted, Reason}}; - {error, Reason} -> - {error, Reason} + Other -> + {error, Other} end. -record_to_row(SUser, SServer, - #offline_msg{from = From, packet = Packet, timestamp = TimeStamp, - expire = Expire, permanent_fields = PermanentFields}) -> - SFrom = mongoose_rdbms:escape_string(jid:to_binary(From)), - SPacket = mongoose_rdbms:escape_string(exml:to_binary(Packet)), - STimeStamp = encode_timestamp(TimeStamp), - SExpire = maybe_encode_timestamp(Expire), - SFields = encode_permanent_fields(PermanentFields), - rdbms_queries:prepare_offline_message(SUser, SServer, STimeStamp, SExpire, SFrom, SPacket, SFields). - -encode_permanent_fields(Fields) -> - Binary = term_to_binary(Fields), - mongoose_rdbms:escape_binary(global, Binary). +count_offline_messages(LUser, LServer, MaxArchivedMsgs) -> + Result = execute_count_offline_messages(LUser, LServer, MaxArchivedMsgs + 1), + mongoose_rdbms:selected_to_integer(Result). remove_user(LUser, LServer) -> - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - rdbms_queries:remove_offline_messages(LServer, SUser, SServer). + execute_offline_delete(LServer, LUser). --spec remove_expired_messages(jid:lserver()) -> {error, term()} | {ok, HowManyRemoved} when - HowManyRemoved :: integer(). +-spec remove_expired_messages(jid:lserver()) -> + {error, term()} | {ok, HowManyRemoved :: non_neg_integer()}. remove_expired_messages(LServer) -> - TimeStamp = erlang:system_time(microsecond), - STimeStamp = encode_timestamp(TimeStamp), - Result = rdbms_queries:remove_expired_offline_messages(LServer, STimeStamp), - case Result of - {error, Reason} -> - {error, Reason}; - {updated, Count} -> - {ok, Count} - end. + TimeStamp = os:system_time(microsecond), + Result = execute_remove_expired_offline_messages(LServer, TimeStamp), + updated_ok(Result). + -spec remove_old_messages(LServer, Timestamp) -> {error, term()} | {ok, HowManyRemoved} when LServer :: jid:lserver(), Timestamp :: integer(), HowManyRemoved :: integer(). remove_old_messages(LServer, TimeStamp) -> - STimeStamp = encode_timestamp(TimeStamp), - Result = rdbms_queries:remove_old_offline_messages(LServer, STimeStamp), - case Result of - {error, Reason} -> - {error, Reason}; - {updated, Count} -> - {ok, Count} - end. + Result = execute_remove_old_offline_messages(LServer, TimeStamp), + updated_ok(Result). + +%% Pure helper functions +record_to_row(LUser, LServer, + #offline_msg{timestamp = TimeStamp, expire = Expire, from = From, + packet = Packet, permanent_fields = PermanentFields}) -> + ExtExpire = maybe_encode_timestamp(Expire), + ExtFrom = jid:to_binary(From), + ExtPacket = exml:to_binary(Packet), + ExtFields = encode_permanent_fields(PermanentFields), + prepare_offline_message(LUser, LServer, TimeStamp, ExtExpire, + ExtFrom, ExtPacket, ExtFields). + +prepare_offline_message(LUser, LServer, ExtTimeStamp, ExtExpire, ExtFrom, ExtPacket, ExtFields) -> + [LUser, LServer, ExtTimeStamp, ExtExpire, ExtFrom, ExtPacket, ExtFields]. -count_offline_messages(LUser, LServer, SUser, SServer, Limit) -> - case rdbms_queries:count_offline_messages(LServer, SUser, SServer, Limit) of - {selected, [{Count}]} -> - mongoose_rdbms:result_to_integer(Count); - Error -> - ?LOG_ERROR(#{what => count_offline_messages_failed, - server => LServer, user => LUser, - reason => Error}), - 0 - end. +encode_permanent_fields(Fields) -> + term_to_binary(Fields). + +maybe_encode_timestamp(never) -> null; +maybe_encode_timestamp(TimeStamp) -> TimeStamp. -encode_timestamp(TimeStamp) -> - mongoose_rdbms:escape_integer(TimeStamp). +rows_to_records(US, To, Rows) -> + [row_to_record(US, To, Row) || Row <- Rows]. + +row_to_record(US, To, {ExtTimeStamp, ExtFrom, ExtPacket, ExtPermanentFields}) -> + {ok, Packet} = exml:parse(ExtPacket), + TimeStamp = mongoose_rdbms:result_to_integer(ExtTimeStamp), + From = jid:from_binary(ExtFrom), + PermanentFields = extract_permanent_fields(ExtPermanentFields), + #offline_msg{us = US, timestamp = TimeStamp, expire = never, + from = From, to = To, packet = Packet, + permanent_fields = PermanentFields}. + +extract_permanent_fields(null) -> + []; %% This is needed in transition period when upgrading to MongooseIM above 3.5.0 +extract_permanent_fields(Escaped) -> + Bin = mongoose_rdbms:unescape_binary(global, Escaped), + binary_to_term(Bin). -maybe_encode_timestamp(never) -> - mongoose_rdbms:escape_null(); -maybe_encode_timestamp(TimeStamp) -> - encode_timestamp(TimeStamp). +updated_ok({updated, Count}) -> {ok, Count}. diff --git a/src/rdbms/mongoose_rdbms.erl b/src/rdbms/mongoose_rdbms.erl index f92c3ca669..156068f7d6 100644 --- a/src/rdbms/mongoose_rdbms.erl +++ b/src/rdbms/mongoose_rdbms.erl @@ -78,9 +78,11 @@ -export([prepare/4, prepared/1, execute/3, + execute_successfully/3, sql_query/2, sql_query_t/1, sql_transaction/2, + transaction_with_delayed_retry/3, sql_dirty/2, to_bool/1, db_engine/1, @@ -115,7 +117,10 @@ use_escaped_null/1]). %% count / integra types decoding --export([result_to_integer/1]). +-export([result_to_integer/1, + selected_to_integer/1]). + +-export([character_to_integer/1]). %% gen_server callbacks -export([init/1, @@ -176,7 +181,8 @@ prepare(Name, Table, [Field | _] = Fields, Statement) when is_atom(Field) -> prepare(Name, Table, [atom_to_binary(F, utf8) || F <- Fields], Statement); prepare(Name, Table, Fields, Statement) when is_atom(Name), is_binary(Table) -> true = lists:all(fun is_binary/1, Fields), - case ets:insert_new(prepared_statements, {Name, Table, Fields, Statement}) of + Tuple = {Name, Table, Fields, iolist_to_binary(Statement)}, + case ets:insert_new(prepared_statements, Tuple) of true -> {ok, Name}; false -> {error, already_exists} end. @@ -190,6 +196,38 @@ prepared(Name) -> execute(Host, Name, Parameters) when is_atom(Name), is_list(Parameters) -> sql_call(Host, {sql_execute, Name, Parameters}). +%% Same as execute/3, but would fail loudly on any error. +-spec execute_successfully(Host :: server(), Name :: atom(), Parameters :: [term()]) -> + query_result(). +execute_successfully(Host, Name, Parameters) -> + try execute(Host, Name, Parameters) of + {selected, _} = Result -> + Result; + {updated, _} = Result -> + Result; + Other -> + Log = #{what => sql_execute_failed, host => Host,statement_name => Name, + statement_query => query_name_to_string(Name), + statement_params => Parameters, reason => Other}, + ?LOG_ERROR(Log), + error(Log) + catch error:Reason:Stacktrace -> + Log = #{what => sql_execute_failed, host => Host, statement_name => Name, + statement_query => query_name_to_string(Name), + statement_params => Parameters, + reason => Reason, stacktrace => Stacktrace}, + ?LOG_ERROR(Log), + erlang:raise(error, Reason, Stacktrace) + end. + +query_name_to_string(Name) -> + case ets:lookup(prepared_statements, Name) of + [] -> + not_found; + [{_, _Table, _Fields, Statement}] -> + Statement + end. + -spec sql_query(Host :: server(), Query :: any()) -> query_result(). sql_query(Host, Query) -> sql_call(Host, {sql_query, Query}). @@ -203,6 +241,32 @@ sql_transaction(Host, Queries) when is_list(Queries) -> sql_transaction(Host, F) when is_function(F) -> sql_call(Host, {sql_transaction, F}). +%% This function allows to specify delay between retries. +-spec transaction_with_delayed_retry(server(), fun() | maybe_improper_list(), map()) -> transaction_result(). +transaction_with_delayed_retry(Host, F, Info) -> + Retries = maps:get(retries, Info), + Delay = maps:get(delay, Info), + do_transaction_with_delayed_retry(Host, F, Retries, Delay, Info). + +do_transaction_with_delayed_retry(Host, F, Retries, Delay, Info) -> + Result = mongoose_rdbms:sql_transaction(Host, F), + case Result of + {atomic, _} -> + Result; + {aborted, Reason} when Retries > 0 -> + ?LOG_WARNING(Info#{what => rdbms_transaction_aborted, + text => <<"Transaction aborted. Restart">>, + reason => Reason, retries_left => Retries}), + timer:sleep(Delay), + do_transaction_with_delayed_retry(Host, F, Retries - 1, Delay, Info); + _ -> + Err = Info#{what => mam_transaction_failed, + text => <<"Transaction failed. Do not restart">>, + reason => Result}, + ?LOG_ERROR(Err), + erlang:error(Err) + end. + -spec sql_dirty(server(), fun()) -> any() | no_return(). sql_dirty(Host, F) when is_function(F) -> case sql_call(Host, {sql_dirty, F}) of @@ -412,6 +476,13 @@ result_to_integer(Int) when is_integer(Int) -> result_to_integer(Bin) when is_binary(Bin) -> binary_to_integer(Bin). +selected_to_integer({selected, [{BInt}]}) -> + result_to_integer(BInt). + +%% Converts a value from a CHAR(1) field to integer +character_to_integer(<>) -> X; +character_to_integer(X) when is_integer(X) -> X. + %% pgsql returns booleans as "t" or "f" -spec to_bool(binary() | string() | atom() | integer() | any()) -> boolean(). to_bool(B) when is_binary(B) -> diff --git a/src/rdbms/mongoose_rdbms_odbc.erl b/src/rdbms/mongoose_rdbms_odbc.erl index 8f12e52f38..f7d4012744 100644 --- a/src/rdbms/mongoose_rdbms_odbc.erl +++ b/src/rdbms/mongoose_rdbms_odbc.erl @@ -75,7 +75,16 @@ query(Connection, Query, Timeout) -> -spec prepare(Connection :: term(), Name :: atom(), Table :: binary(), Fields :: [binary()], Statement :: iodata()) -> {ok, {binary(), [fun((term()) -> tuple())]}}. -prepare(Connection, _Name, Table, Fields, Statement) -> +prepare(Connection, Name, Table, Fields, Statement) -> + try prepare2(Connection, Table, Fields, Statement) + catch Class:Reason:Stacktrace -> + ?LOG_ERROR(#{what => prepare_failed, + statement_name => Name, sql_query => Statement, + class => Class, reason => Reason, stacktrace => Stacktrace}), + erlang:raise(Class, Reason, Stacktrace) + end. + +prepare2(Connection, Table, Fields, Statement) -> {ok, TableDesc} = eodbc:describe_table(Connection, unicode:characters_to_list(Table)), ServerType = server_type(), ParamMappers = [field_name_to_mapper(ServerType, TableDesc, Field) || Field <- Fields], @@ -153,8 +162,10 @@ parse_row([], []) -> FieldName :: binary()) -> fun((term()) -> tuple()). field_name_to_mapper(_ServerType, _TableDesc, <<"limit">>) -> fun(P) -> {sql_integer, [P]} end; +field_name_to_mapper(_ServerType, _TableDesc, <<"offset">>) -> + fun(P) -> {sql_integer, [P]} end; field_name_to_mapper(_ServerType, TableDesc, FieldName) -> - {_, ODBCType} = lists:keyfind(unicode:characters_to_list(FieldName), 1, TableDesc), + ODBCType = field_to_odbc_type(unicode:characters_to_list(FieldName), TableDesc), case simple_type(just_type(ODBCType)) of binary -> fun(P) -> binary_mapper(P) end; @@ -166,6 +177,16 @@ field_name_to_mapper(_ServerType, TableDesc, FieldName) -> fun(P) -> {ODBCType, [P]} end end. +field_to_odbc_type(FieldName, TableDesc) -> + case lists:keyfind(FieldName, 1, TableDesc) of + false -> + ?LOG_ERROR(#{what => field_to_odbc_type_failed, + field => FieldName, table_desc => TableDesc}), + error(field_to_odbc_type_failed); + {_, ODBCType} -> + ODBCType + end. + unicode_mapper(P) -> Utf16 = unicode_characters_to_binary(iolist_to_binary(P), utf8, {utf16, little}), Len = byte_size(Utf16) div 2, diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index 37d241d8da..ad4edc4f80 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -28,13 +28,14 @@ -export([get_db_type/0, begin_trans/0, - get_db_specific_limits/1, + get_db_specific_limits/0, + get_db_specific_limits_binaries/0, + get_db_specific_limits_binaries/1, get_db_specific_offset/2, + add_limit_arg/2, + limit_offset_sql/0, + limit_offset_args/2, sql_transaction/2, - get_last/2, - select_last/3, - set_last_t/4, - del_last/2, get_password/2, set_password_t/3, add_user/3, @@ -46,51 +47,7 @@ users_number/2, get_users_without_scram/2, get_users_without_scram_count/1, - get_average_roster_size/1, - get_average_rostergroup_size/1, - clear_rosters/1, - get_roster/2, - get_roster_jid_groups/2, - get_roster_groups/3, - del_user_roster_t/2, - get_roster_by_jid/3, - get_roster_by_jid_t/3, - get_rostergroup_by_jid/3, - get_rostergroup_by_jid_t/3, - del_roster/3, - del_roster_sql/2, - update_roster/5, - update_roster_sql/4, - roster_subscribe/4, - get_subscription/3, - get_subscription_t/3, - get_default_privacy_list/2, - get_default_privacy_list_t/1, - count_privacy_lists/1, - clear_privacy_lists/1, - get_privacy_list_names/2, - get_privacy_list_names_t/1, - get_privacy_list_id/3, - get_privacy_list_id_t/2, - get_privacy_list_data/3, - get_privacy_list_data_by_id/2, - set_default_privacy_list/2, - unset_default_privacy_list/2, - remove_privacy_list/2, - add_privacy_list/2, - set_privacy_list/2, - del_privacy_lists/3, count_records_where/3, - get_roster_version/2, - set_roster_version/2, - prepare_offline_message/7, - push_offline_messages/2, - pop_offline_messages/4, - fetch_offline_messages/4, - count_offline_messages/4, - remove_old_offline_messages/2, - remove_expired_offline_messages/2, - remove_offline_messages/3, create_bulk_insert_query/3]). -export([join/2, @@ -149,24 +106,6 @@ update_t(Table, Fields, Vals, Where) -> join_escaped(Vals) -> join([mongoose_rdbms:use_escaped(X) || X <- Vals], ", "). - -update(LServer, Table, Fields, Vals, Where) -> - UPairs = lists:zipwith(fun(A, B) -> [A, "=", mongoose_rdbms:use_escaped(B)] end, - Fields, Vals), - case mongoose_rdbms:sql_query( - LServer, - [<<"update ">>, Table, <<" set ">>, - join(UPairs, ", "), - <<" where ">>, Where, ";"]) of - {updated, 1} -> - ok; - _ -> - mongoose_rdbms:sql_query( - LServer, - [<<"insert into ">>, Table, "(", join(Fields, ", "), - <<") values (">>, join_escaped(Vals), ");"]) - end. - -spec execute_upsert(Host :: mongoose_rdbms:server(), Name :: atom(), InsertParams :: [any()], @@ -279,30 +218,6 @@ begin_trans(mssql) -> begin_trans(_) -> [<<"BEGIN;">>]. - -get_last(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select seconds, state from last " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -select_last(LServer, TStamp, Comparator) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select username, seconds, state from last " - "where seconds ">>, Comparator, " ", - mongoose_rdbms:use_escaped_integer(mongoose_rdbms:escape_integer(TStamp)), ";"]). - -set_last_t(LServer, Username, Seconds, State) -> - update(LServer, "last", ["username", "seconds", "state"], - [Username, Seconds, State], - [<<"username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -del_last(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from last where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - get_password(LServer, Username) -> mongoose_rdbms:sql_query( LServer, @@ -433,374 +348,12 @@ get_users_without_scram_count(LServer) -> LServer, [<<"select count(*) from users where pass_details is null">>]). -get_average_roster_size(Server) -> - mongoose_rdbms:sql_query( - Server, - [<<"select avg(items) from " - "(select count(*) as items from rosterusers group by username) as items;">>]). - -get_average_rostergroup_size(Server) -> - mongoose_rdbms:sql_query( - Server, - [<<"select avg(roster) from " - "(select count(*) as roster from rostergroups group by username) as roster;">>]). - -clear_rosters(Server) -> - mongoose_rdbms:sql_transaction( - Server, - fun() -> - mongoose_rdbms:sql_query_t( - [<<"delete from rosterusers;">>]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups;">>]) - end). - -get_roster(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select username, jid, nick, subscription, ask, " - "askmessage, server, subscribe, type from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_roster_jid_groups(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select jid, grp from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_roster_groups(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t( - [<<"select grp from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID), ";"]). - -del_user_roster_t(LServer, Username) -> - mongoose_rdbms:sql_transaction( - LServer, - fun() -> - mongoose_rdbms:sql_query_t( - [<<"delete from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]) - end). - -q_get_roster(Username, SJID) -> - [<<"select username, jid, nick, subscription, " - "ask, askmessage, server, subscribe, type from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. - -get_roster_by_jid(LServer, Username, SJID) -> - mongoose_rdbms:sql_query(LServer, q_get_roster(Username, SJID)). - -get_roster_by_jid_t(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t(q_get_roster(Username, SJID)). - -q_get_rostergroup(Username, SJID) -> - [<<"select grp from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. - -get_rostergroup_by_jid(LServer, Username, SJID) -> - mongoose_rdbms:sql_query(LServer, q_get_rostergroup(Username, SJID)). - -get_rostergroup_by_jid_t(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t(q_get_rostergroup(Username, SJID)). - -del_roster(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t( - [<<"delete from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]). - -del_roster_sql(Username, SJID) -> - [[<<"delete from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)], - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]]. - -update_roster(_LServer, Username, SJID, ItemVals, ItemGroups) -> - update_t(<<"rosterusers">>, - [<<"username">>, <<"jid">>, <<"nick">>, <<"subscription">>, <<"ask">>, - <<"askmessage">>, <<"server">>, <<"subscribe">>, <<"type">>], - ItemVals, - [<<"username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), - lists:foreach(fun(ItemGroup) -> - mongoose_rdbms:sql_query_t( - [<<"insert into rostergroups(username, jid, grp) " - "values (">>, join_escaped(ItemGroup), ");"]) - end, - ItemGroups). - -update_roster_sql(Username, SJID, ItemVals, ItemGroups) -> - [[<<"delete from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)], - [<<"insert into rosterusers(" - "username, jid, nick, " - "subscription, ask, askmessage, " - "server, subscribe, type) " - " values (">>, join_escaped(ItemVals), ");"], - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID), ";"]] ++ - [[<<"insert into rostergroups(username, jid, grp) " - "values (">>, join_escaped(ItemGroup), ");"] || - ItemGroup <- ItemGroups]. - -roster_subscribe(_LServer, Username, SJID, ItemVals) -> - update_t(<<"rosterusers">>, - [<<"username">>, <<"jid">>, <<"nick">>, <<"subscription">>, <<"ask">>, - <<"askmessage">>, <<"server">>, <<"subscribe">>, <<"type">>], - ItemVals, - [<<"username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]). - -q_get_subscription(Username, SJID) -> - [<<"select subscription from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. - -get_subscription(LServer, Username, SJID) -> - mongoose_rdbms:sql_query( LServer, q_get_subscription(Username, SJID)). - -get_subscription_t(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t(q_get_subscription(Username, SJID)). - -get_default_privacy_list(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select name from privacy_default_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_default_privacy_list_t(Username) -> - mongoose_rdbms:sql_query_t( - [<<"select name from privacy_default_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -count_privacy_lists(LServer) -> - mongoose_rdbms:sql_query(LServer, [<<"select count(*) from privacy_list;">>]). - -clear_privacy_lists(LServer) -> - mongoose_rdbms:sql_query(LServer, [<<"delete from privacy_list;">>]). - -get_privacy_list_names(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select name from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_privacy_list_names_t(Username) -> - mongoose_rdbms:sql_query_t( - [<<"select name from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_privacy_list_id(LServer, Username, SName) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select id from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and name=">>, mongoose_rdbms:use_escaped_string(SName)]). - -get_privacy_list_id_t(Username, SName) -> - mongoose_rdbms:sql_query_t( - [<<"select id from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and name=">>, mongoose_rdbms:use_escaped_string(SName)]). - -get_privacy_list_data(LServer, Username, SName) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select t, value, action, ord, match_all, match_iq, " - "match_message, match_presence_in, match_presence_out " - "from privacy_list_data " - "where id = (select id from privacy_list where " - "username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and name=">>, mongoose_rdbms:use_escaped_string(SName), <<") " - "order by ord;">>]). - -get_privacy_list_data_by_id(LServer, ID) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select t, value, action, ord, match_all, match_iq, " - "match_message, match_presence_in, match_presence_out " - "from privacy_list_data " - "where id=">>, mongoose_rdbms:use_escaped_integer(ID), <<" order by ord;">>]). - -set_default_privacy_list(Username, SName) -> - update_t(<<"privacy_default_list">>, [<<"username">>, <<"name">>], - [Username, SName], - [<<"username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -unset_default_privacy_list(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from privacy_default_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -remove_privacy_list(Username, SName) -> - mongoose_rdbms:sql_query_t( - [<<"delete from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), - " and name=", mongoose_rdbms:use_escaped_string(SName)]). - -add_privacy_list(Username, SName) -> - mongoose_rdbms:sql_query_t( - [<<"insert into privacy_list(username, name) " - "values (">>, mongoose_rdbms:use_escaped_string(Username), - ", ", mongoose_rdbms:use_escaped_string(SName), ");"]). - --spec set_privacy_list(mongoose_rdbms:escaped_integer(), - list(list(mongoose_rdbms:escaped_value()))) -> ok. -set_privacy_list(ID, RItems) -> - mongoose_rdbms:sql_query_t( - [<<"delete from privacy_list_data " - "where id=">>, mongoose_rdbms:use_escaped_integer(ID), ";"]), - lists:foreach(fun(Items) -> - mongoose_rdbms:sql_query_t( - [<<"insert into privacy_list_data(" - "id, t, value, action, ord, match_all, match_iq, " - "match_message, match_presence_in, " - "match_presence_out " - ") " - "values (">>, mongoose_rdbms:use_escaped_integer(ID), ", ", - join_escaped(Items), ");"]) - end, RItems). - -del_privacy_lists(LServer, _Server, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from privacy_list_data where id in " - "( select id from privacy_list as pl where pl.username=">>, - mongoose_rdbms:use_escaped_string(Username), <<";">>]), - mongoose_rdbms:sql_query( - LServer, - [<<"delete from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]), - mongoose_rdbms:sql_query( - LServer, - [<<"delete from privacy_default_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - %% Count number of records in a table given a where clause count_records_where(LServer, Table, WhereClause) -> mongoose_rdbms:sql_query( LServer, [<<"select count(*) from ">>, Table, " ", WhereClause, ";"]). - -get_roster_version(LServer, LUser) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select version from roster_version " - "where username=">>, mongoose_rdbms:use_escaped_string(LUser)]). - -set_roster_version(LUser, Version) -> - update_t( - <<"roster_version">>, - [<<"username">>, <<"version">>], - [LUser, Version], - [<<"username = ">>, mongoose_rdbms:use_escaped_string(LUser)]). - - -pop_offline_messages(LServer, SUser, SServer, STimeStamp) -> - SelectSQL = select_offline_messages_sql(SUser, SServer, STimeStamp), - DeleteSQL = delete_offline_messages_sql(SUser, SServer), - F = fun() -> - Res = mongoose_rdbms:sql_query_t(SelectSQL), - mongoose_rdbms:sql_query_t(DeleteSQL), - Res - end, - mongoose_rdbms:sql_transaction(LServer, F). - -fetch_offline_messages(LServer, SUser, SServer, STimeStamp) -> - mongoose_rdbms:sql_query(LServer, select_offline_messages_sql(SUser, SServer, STimeStamp)). - -select_offline_messages_sql(SUser, SServer, STimeStamp) -> - [<<"select timestamp, from_jid, packet, permanent_fields from offline_message " - "where server = ">>, mongoose_rdbms:use_escaped_string(SServer), <<" and " - "username = ">>, mongoose_rdbms:use_escaped_string(SUser), <<" and " - "(expire is null or expire > ">>, mongoose_rdbms:use_escaped_integer(STimeStamp), <<") " - "ORDER BY timestamp">>]. - -delete_offline_messages_sql(SUser, SServer) -> - [<<"delete from offline_message " - "where server = ">>, mongoose_rdbms:use_escaped_string(SServer), <<" and " - "username = ">>, mongoose_rdbms:use_escaped_string(SUser)]. - -remove_old_offline_messages(LServer, STimeStamp) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from offline_message where timestamp < ">>, - mongoose_rdbms:use_escaped_integer(STimeStamp)]). - -remove_expired_offline_messages(LServer, STimeStamp) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from offline_message " - "where expire is not null and expire < ">>, - mongoose_rdbms:use_escaped_integer(STimeStamp)]). - -remove_offline_messages(LServer, SUser, SServer) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from offline_message " - "where server = ">>, mongoose_rdbms:use_escaped_string(SServer), <<" and " - "username = ">>, mongoose_rdbms:use_escaped_string(SUser)]). - --spec prepare_offline_message(SUser, SServer, STimeStamp, SExpire, SFrom, SPacket, SFields) -> - mongoose_rdbms:sql_query_part() when - SUser :: mongoose_rdbms:escaped_string(), - SServer :: mongoose_rdbms:escaped_string(), - STimeStamp :: mongoose_rdbms:escaped_timestamp(), - SExpire :: mongoose_rdbms:escaped_timestamp() | mongoose_rdbms:escaped_null(), - SFrom :: mongoose_rdbms:escaped_string(), - SPacket :: mongoose_rdbms:escaped_string(), - SFields :: mongoose_rdbms:escaped_binary(). -prepare_offline_message(SUser, SServer, STimeStamp, SExpire, SFrom, SPacket, SFields) -> - [<<"(">>, mongoose_rdbms:use_escaped_string(SUser), - <<", ">>, mongoose_rdbms:use_escaped_string(SServer), - <<", ">>, mongoose_rdbms:use_escaped_integer(STimeStamp), - <<", ">>, mongoose_rdbms:use_escaped(SExpire), - <<", ">>, mongoose_rdbms:use_escaped_string(SFrom), - <<", ">>, mongoose_rdbms:use_escaped_string(SPacket), - <<", ">>, mongoose_rdbms:use_escaped_binary(SFields), - <<")">>]. - -push_offline_messages(LServer, Rows) -> - mongoose_rdbms:sql_query( - LServer, - [<<"INSERT INTO offline_message " - "(username, server, timestamp, expire, from_jid, packet, permanent_fields) " - "VALUES ">>, join(Rows, ", ")]). - - -count_offline_messages(LServer, SUser, SServer, Limit) -> - count_offline_messages(?RDBMS_TYPE, LServer, SUser, SServer, Limit). - -count_offline_messages(mssql, LServer, SUser, SServer, Limit) -> - rdbms_queries_mssql:count_offline_messages(LServer, SUser, SServer, Limit); -count_offline_messages(_, LServer, SUser, SServer, Limit) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select count(*) from offline_message " - "where server = ">>, mongoose_rdbms:use_escaped_string(SServer), <<" and " - "username = ">>, mongoose_rdbms:use_escaped_string(SUser), <<" " - "limit ">>, integer_to_list(Limit)]). - -spec create_bulk_insert_query(Table :: iodata() | atom(), Fields :: [iodata() | atom()], RowsNum :: pos_integer()) -> {iodata(), [binary()]}. @@ -819,20 +372,33 @@ create_bulk_insert_query(Table, Fields, RowsNum) when RowsNum > 0 -> Fields2 = lists:append(lists:duplicate(RowsNum, Fields)), {Sql, Fields2}. +get_db_specific_limits() -> + do_get_db_specific_limits(?RDBMS_TYPE, "?", true). + +get_db_specific_limits_binaries() -> + {LimitSQL, LimitMSSQL} = get_db_specific_limits(), + {list_to_binary(LimitSQL), list_to_binary(LimitMSSQL)}. + -spec get_db_specific_limits(integer()) -> {SQL :: nonempty_string(), []} | {[], MSSQL::nonempty_string()}. get_db_specific_limits(Limit) -> LimitStr = integer_to_list(Limit), - do_get_db_specific_limits(?RDBMS_TYPE, LimitStr). + do_get_db_specific_limits(?RDBMS_TYPE, LimitStr, false). -spec get_db_specific_offset(integer(), integer()) -> iolist(). get_db_specific_offset(Offset, Limit) -> do_get_db_specific_offset(?RDBMS_TYPE, integer_to_list(Offset), integer_to_list(Limit)). -do_get_db_specific_limits(mssql, LimitStr) -> +%% Arguments: +%% - Type (atom) - database type +%% - LimitStr (string) - a field value +%% - Wrap (boolean) - add parentheses around a field for MSSQL +do_get_db_specific_limits(mssql, LimitStr, _Wrap = false) -> {"", "TOP " ++ LimitStr}; -do_get_db_specific_limits(_, LimitStr) -> +do_get_db_specific_limits(mssql, LimitStr, _Wrap = true) -> + {"", "TOP (" ++ LimitStr ++ ")"}; +do_get_db_specific_limits(_, LimitStr, _Wrap) -> {"LIMIT " ++ LimitStr, ""}. do_get_db_specific_offset(mssql, Offset, Limit) -> @@ -840,3 +406,29 @@ do_get_db_specific_offset(mssql, Offset, Limit) -> " FETCH NEXT ", Limit, " ROWS ONLY"]; do_get_db_specific_offset(_, Offset, _Limit) -> [" OFFSET ", Offset]. + +add_limit_arg(Limit, Args) -> + add_limit_arg(?RDBMS_TYPE, Limit, Args). + +add_limit_arg(mssql, Limit, Args) -> + [Limit|Args]; +add_limit_arg(_, Limit, Args) -> + Args ++ [Limit]. + +get_db_specific_limits_binaries(Limit) -> + {LimitSQL, LimitMSSQL} = get_db_specific_limits(Limit), + {list_to_binary(LimitSQL), list_to_binary(LimitMSSQL)}. + +limit_offset_sql() -> + limit_offset_sql(?RDBMS_TYPE). + +limit_offset_sql(mssql) -> + <<" OFFSET (?) ROWS FETCH NEXT (?) ROWS ONLY">>; +limit_offset_sql(_) -> + <<" LIMIT ? OFFSET ?">>. + +limit_offset_args(Limit, Offset) -> + limit_offset_args(?RDBMS_TYPE, Limit, Offset). + +limit_offset_args(mssql, Limit, Offset) -> [Offset, Limit]; +limit_offset_args(_, Limit, Offset) -> [Limit, Offset]. diff --git a/src/rdbms/rdbms_queries_mssql.erl b/src/rdbms/rdbms_queries_mssql.erl index 0f8eba352c..64b05ea251 100644 --- a/src/rdbms/rdbms_queries_mssql.erl +++ b/src/rdbms/rdbms_queries_mssql.erl @@ -26,26 +26,8 @@ -include("mongoose.hrl"). %% API --export([begin_trans/0, - query_archive_id/3, - count_offline_messages/4]). +-export([begin_trans/0]). begin_trans() -> [<<"BEGIN TRANSACTION;">>]. - -query_archive_id(Host, SServer, SUserName) -> - mongoose_rdbms:sql_query( - Host, - ["SELECT TOP 1 id " - "FROM mam_server_user " - "WHERE server=", mongoose_rdbms:use_escaped_string(SServer), - " AND user_name=", mongoose_rdbms:use_escaped_string(SUserName)]). - -count_offline_messages(LServer, SUser, SServer, Limit) -> - mongoose_rdbms:sql_query( - LServer, - [<<"SELECT TOP ">>, integer_to_list(Limit), - <<"count(*) FROM offline_message " - "WHERE server=">>, mongoose_rdbms:use_escaped_string(SServer), - <<" AND username=">>, mongoose_rdbms:use_escaped_string(SUser)]).