From 85811dbd7461fd9a2bd40c39615e85e07e0a5deb Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 25 Nov 2020 09:12:12 +0100 Subject: [PATCH 01/92] Use prepared queries for message retraction --- src/mam/mod_mam_rdbms_arch.erl | 74 +++++++++++++++++++--------------- src/rdbms/rdbms_queries.erl | 27 +++++++++++-- 2 files changed, 65 insertions(+), 36 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 5a08a96401..97c9294e21 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -75,6 +75,21 @@ start(Host, Opts) -> [<<"SELECT COUNT(*) FROM mam_message ">>, index_hint_sql(Host), <<"WHERE user_id = ?">>]), + mongoose_rdbms:prepare(mam_archive_remove, mam_message, [user_id], + [<<"DELETE FROM mam_message " + "WHERE user_id = ?">>]), + mongoose_rdbms:prepare(mam_make_tombstone, mam_message, [message, user_id, id], + [<<"UPDATE mam_message SET message = ?, search_body = '' " + "WHERE user_id = ? AND id = ?">>]), + + {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(1), + mongoose_rdbms:prepare(mam_select_messages_to_retract, mam_message, + [user_id, remote_bare_jid, origin_id, direction], + [<<"SELECT ", LimitMSSQL/binary, + " id, message FROM mam_message" + " WHERE user_id = ? AND remote_bare_jid = ? " + " AND origin_id = ? AND direction = ?" + " ORDER BY id DESC ", LimitSQL/binary>>]), ok. @@ -172,46 +187,40 @@ retract_message(Host, #{archive_id := UserID, direction := Dir, packet := Packet}) -> case mod_mam_utils:get_retract_id(mod_mam, Host, Packet) of - none -> ok; - OriginIDToRetract -> retract_message(Host, UserID, LocJID, RemJID, OriginIDToRetract, Dir) + none -> + ok; + OriginIDToRetract -> + retract_message(Host, UserID, LocJID, RemJID, OriginIDToRetract, Dir) end. retract_message(Host, UserID, LocJID, RemJID, OriginID, Dir) -> - SUserID = use_escaped_integer(escape_user_id(UserID)), - SOriginID = use_escaped_string(escape_string(OriginID)), - SBareRemJID = use_escaped_string(minify_and_escape_bare_jid(Host, LocJID, RemJID)), - SDir = encode_direction(Dir), - Query = query_for_messages_to_retract(SUserID, SBareRemJID, SOriginID, SDir), - {selected, Rows} = mod_mam_utils:success_sql_query(Host, Query), - make_tombstone(Host, SUserID, OriginID, Rows), + MinBareRemJID = minify_bare_jid(Host, LocJID, RemJID), + BinDir = encode_direction(Dir), + {selected, Rows} = execute_select_messages_to_retract( + Host, UserID, MinBareRemJID, OriginID, BinDir), + make_tombstone(Host, UserID, OriginID, Rows), ok. -make_tombstone(_Host, SUserID, OriginID, []) -> +make_tombstone(_Host, UserID, OriginID, []) -> ?LOG_INFO(#{what => make_tombstone_failed, text => <<"Message to retract was not found by origin id">>, - user_id => SUserID, origin_id => OriginID}); -make_tombstone(Host, SUserID, OriginID, [{ResMessID, ResData}]) -> + user_id => UserID, origin_id => OriginID}); +make_tombstone(Host, UserID, OriginID, [{ResMessID, ResData}]) -> Data = mongoose_rdbms:unescape_binary(Host, ResData), Packet = stored_binary_to_packet(Host, Data), MessID = mongoose_rdbms:result_to_integer(ResMessID), Tombstone = mod_mam_utils:tombstone(Packet, OriginID), TombstoneData = packet_to_stored_binary(Host, Tombstone), - STombstoneData = mongoose_rdbms:use_escaped_binary( - mongoose_rdbms:escape_binary(Host, TombstoneData)), - BMessID = use_escaped_integer(escape_message_id(MessID)), - UpdateQuery = query_to_make_tombstone(STombstoneData, SUserID, BMessID), - {updated, 1} = mod_mam_utils:success_sql_query(Host, UpdateQuery). - -query_for_messages_to_retract(SUserID, SBareRemJID, SOriginID, SDir) -> - {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits(1), - ["SELECT ", LimitMSSQL, " id, message FROM mam_message" - " WHERE user_id = ", SUserID, " AND remote_bare_jid = ", SBareRemJID, - " AND origin_id = ", SOriginID, " AND direction = '", SDir, "'" - " ORDER BY id DESC ", LimitSQL]. - -query_to_make_tombstone(STombstoneData, SUserID, BMessID) -> - ["UPDATE mam_message SET message = ", STombstoneData, ", search_body = ''" - " WHERE user_id = ", SUserID, " AND id = '", BMessID, "'"]. + execute_make_tombstone(Host, TombstoneData, UserID, MessID). + +execute_select_messages_to_retract(Host, UserID, BareRemJID, OriginID, Dir) -> + mod_mam_utils:success_sql_execute(Host, mam_select_messages_to_retract, + [UserID, BareRemJID, OriginID, Dir]). + +execute_make_tombstone(Host, TombstoneData, UserID, MessID) -> + {updated, _} = + mod_mam_utils:success_sql_execute(Host, mam_make_tombstone, + [TombstoneData, UserID, MessID]). -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). prepare_message(Host, #{message_id := MessID, @@ -460,11 +469,7 @@ remove_archive(Acc, Host, UserID, _UserJID) -> Acc. remove_archive(Host, UserID) -> - {updated, _} = - mod_mam_utils:success_sql_query( - Host, - ["DELETE FROM mam_message " - "WHERE user_id = ", use_escaped_integer(escape_user_id(UserID))]). + {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_archive_remove, [UserID]). %% @doc Each record is a tuple of form %% `{<<"13663125233">>, <<"bob@localhost">>, <>}'. @@ -666,6 +671,9 @@ escape_user_id(UserID) when is_integer(UserID) -> minify_and_escape_bare_jid(Host, LocJID, JID) -> escape_string(jid_to_stored_binary(Host, LocJID, jid:to_bare(JID))). +minify_bare_jid(Host, LocJID, JID) -> + jid_to_stored_binary(Host, LocJID, jid:to_bare(JID)). + maybe_encode_compact_uuid(undefined, _) -> undefined; maybe_encode_compact_uuid(Microseconds, NodeID) -> diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index 37d241d8da..998ae45e34 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -28,8 +28,11 @@ -export([get_db_type/0, begin_trans/0, + get_db_specific_limits/0, get_db_specific_limits/1, + get_db_specific_limits_binaries/1, get_db_specific_offset/2, + add_limit_arg/2, sql_transaction/2, get_last/2, select_last/3, @@ -823,16 +826,18 @@ create_bulk_insert_query(Table, Fields, RowsNum) when RowsNum > 0 -> -> {SQL :: nonempty_string(), []} | {[], MSSQL::nonempty_string()}. get_db_specific_limits(Limit) -> LimitStr = integer_to_list(Limit), - do_get_db_specific_limits(?RDBMS_TYPE, LimitStr). + do_get_db_specific_limits(?RDBMS_TYPE, LimitStr, false). -spec get_db_specific_offset(integer(), integer()) -> iolist(). get_db_specific_offset(Offset, Limit) -> do_get_db_specific_offset(?RDBMS_TYPE, integer_to_list(Offset), integer_to_list(Limit)). -do_get_db_specific_limits(mssql, LimitStr) -> +do_get_db_specific_limits(mssql, LimitStr, false) -> {"", "TOP " ++ LimitStr}; -do_get_db_specific_limits(_, LimitStr) -> +do_get_db_specific_limits(mssql, LimitStr, true) -> + {"", "TOP (" ++ LimitStr ++ ")"}; +do_get_db_specific_limits(_, LimitStr, _Wrap) -> {"LIMIT " ++ LimitStr, ""}. do_get_db_specific_offset(mssql, Offset, Limit) -> @@ -840,3 +845,19 @@ do_get_db_specific_offset(mssql, Offset, Limit) -> " FETCH NEXT ", Limit, " ROWS ONLY"]; do_get_db_specific_offset(_, Offset, _Limit) -> [" OFFSET ", Offset]. + +get_db_specific_limits() -> + LimitStr = "?", + do_get_db_specific_limits(?RDBMS_TYPE, LimitStr, true). + +add_limit_arg(Limit, Args) -> + add_limit_arg(?RDBMS_TYPE, Limit, Args). + +add_limit_arg(mssql, Limit, Args) -> + [Limit|Args]; +add_limit_arg(_, Limit, Args) -> + Args ++ [Limit]. + +get_db_specific_limits_binaries(Limit) -> + {LimitSQL, LimitMSSQL} = get_db_specific_limits(Limit), + {list_to_binary(LimitSQL), list_to_binary(LimitMSSQL)}. From 50fae37cdaad9659f2e6fe787b74a512826714d3 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 27 Nov 2020 10:17:22 +0100 Subject: [PATCH 02/92] Make lookups prepared in mod_mam --- src/mam/mod_mam_rdbms_arch.erl | 318 +++++++++++++++++++----------- src/rdbms/mongoose_rdbms_odbc.erl | 2 + src/rdbms/rdbms_queries.erl | 18 ++ 3 files changed, 222 insertions(+), 116 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 97c9294e21..86b0391d8f 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -56,7 +56,8 @@ %% ---------------------------------------------------------------------- %% Types --type filter() :: mongoose_rdbms:sql_query_part(). +-type filter_field() :: tuple(). +-type filter() :: [filter_field()]. -type escaped_message_id() :: mongoose_rdbms:escape_string(). -type escaped_jid() :: mongoose_rdbms:escaped_string(). -type escaped_resource() :: mongoose_rdbms:escaped_string(). @@ -357,7 +358,7 @@ lookup_messages_opt_count(Host, UserJID, false -> IndexHintSQL = index_hint_sql(Host), FirstID = row_to_message_id(hd(MessageRows)), - Offset = calc_count(Host, before_id(FirstID, Filter), IndexHintSQL), + Offset = calc_count(Host, before_id(FirstID, Filter)), {ok, {Offset + MessageRowsCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}} end; @@ -374,7 +375,7 @@ lookup_messages_opt_count(Host, UserJID, false -> IndexHintSQL = index_hint_sql(Host), LastID = row_to_message_id(lists:last(MessageRows)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter), IndexHintSQL), + CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), {ok, {Offset + MessageRowsCount + CountAfterLastID, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}} end; @@ -391,7 +392,7 @@ lookup_messages_opt_count(Host, UserJID, false -> IndexHintSQL = index_hint_sql(Host), LastID = row_to_message_id(lists:last(MessageRows)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter), IndexHintSQL), + CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), {ok, {MessageRowsCount + CountAfterLastID, 0, rows_to_uniform_format(Host, UserJID, MessageRows)}} end. @@ -399,51 +400,44 @@ lookup_messages_opt_count(Host, UserJID, lookup_messages_regular(Host, UserJID, RSM = #rsm_in{direction = aft, id = ID}, PageSize, Filter) when ID =/= undefined -> - IndexHintSQL = index_hint_sql(Host), - TotalCount = calc_count(Host, Filter, IndexHintSQL), - Offset = calc_offset(Host, Filter, IndexHintSQL, PageSize, TotalCount, RSM), + TotalCount = calc_count(Host, Filter), + Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), MessageRows = extract_messages(Host, from_id(ID, Filter), 0, PageSize + 1, false), Result = {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}, mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); lookup_messages_regular(Host, UserJID, RSM = #rsm_in{direction = before, id = ID}, PageSize, Filter) when ID =/= undefined -> - IndexHintSQL = index_hint_sql(Host), - TotalCount = calc_count(Host, Filter, IndexHintSQL), - Offset = calc_offset(Host, Filter, IndexHintSQL, PageSize, TotalCount, RSM), + TotalCount = calc_count(Host, Filter), + Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), MessageRows = extract_messages(Host, to_id(ID, Filter), 0, PageSize + 1, true), Result = {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}, mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); lookup_messages_regular(Host, UserJID, RSM, PageSize, Filter) -> - IndexHintSQL = index_hint_sql(Host), - TotalCount = calc_count(Host, Filter, IndexHintSQL), - Offset = calc_offset(Host, Filter, IndexHintSQL, PageSize, TotalCount, RSM), + TotalCount = calc_count(Host, Filter), + Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), {ok, {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}}. -spec after_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). after_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id > ", use_escaped_integer(SID)]. + [{greater, id, ID}|Filter]. -spec before_id(ID :: escaped_message_id() | undefined, Filter :: filter()) -> filter(). before_id(undefined, Filter) -> Filter; before_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id < ", use_escaped_integer(SID)]. + [{lower, id, ID}|Filter]. -spec from_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). from_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id >= ", use_escaped_integer(SID)]. + [{ge, id, ID}|Filter]. -spec to_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). to_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id <= ", use_escaped_integer(SID)]. + [{le, id, ID}|Filter]. rows_to_uniform_format(Host, UserJID, MessageRows) -> [do_row_to_uniform_format(Host, UserJID, Row) || Row <- MessageRows]. @@ -482,36 +476,149 @@ extract_messages(_Host, _Filter, _IOffset, 0, _) -> []; extract_messages(Host, Filter, IOffset, IMax, false) -> {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, " ORDER BY id "), + do_extract_messages(Host, Filter, IOffset, IMax, asc), ?LOG_DEBUG(#{what => mam_extract_messages, mam_filter => Filter, offset => IOffset, max => IMax, host => Host, message_rows => MessageRows}), MessageRows; extract_messages(Host, Filter, IOffset, IMax, true) -> {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, " ORDER BY id DESC "), + do_extract_messages(Host, Filter, IOffset, IMax, desc), ?LOG_DEBUG(#{what => mam_extract_messages, mam_filter => Filter, offset => IOffset, max => IMax, host => Host, message_rows => MessageRows}), lists:reverse(MessageRows). -do_extract_messages(Host, Filter, 0, IMax, Order) -> - {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits(IMax), - mod_mam_utils:success_sql_query( - Host, - ["SELECT ", LimitMSSQL, " id, from_jid, message " - "FROM mam_message ", - Filter, - Order, - " ", LimitSQL]); -do_extract_messages(Host, Filter, IOffset, IMax, Order) -> - {LimitSQL, _LimitMSSQL} = rdbms_queries:get_db_specific_limits(IMax), - Offset = rdbms_queries:get_db_specific_offset(IOffset, IMax), - mod_mam_utils:success_sql_query( - Host, - ["SELECT id, from_jid, message " - "FROM mam_message ", - Filter, Order, LimitSQL, Offset]). +do_extract_messages(Host, Filters, IOffset, IMax, Order) -> + Filters2 = Filters ++ rdbms_queries:limit_offset_filters(IMax, IOffset), + do_lookup_query(lookup, Host, Filters2, Order). + +do_lookup_query(QueryType, Host, Filters, Order) -> + StmtName = filters_to_statement_name(QueryType, Filters, Order), + case mongoose_rdbms:prepared(StmtName) of + false -> + %% Create a new type of a query + SQL = lookup_sql_binary(QueryType, Filters, Order), + Columns = filters_to_columns(Filters), + mongoose_rdbms:prepare(StmtName, mam_message, Columns, SQL); + true -> + ok + end, + Args = filters_to_args(Filters), + try mongoose_rdbms:execute(Host, StmtName, Args) of + {selected, Rs} when is_list(Rs) -> + {selected, Rs}; + Error -> + What = #{what => mam_lookup_failed, statement => StmtName, + sql_query => lookup_sql_binary(QueryType, Filters, Order), + reason => Error, host => Host}, + ?LOG_ERROR(What), + error(What) + catch Class:Error:Stacktrace -> + What = #{what => mam_lookup_failed, statement => StmtName, + sql_query => lookup_sql_binary(QueryType, Filters, Order), + class => Class, stacktrace => Stacktrace, + reason => Error, host => Host}, + ?LOG_ERROR(What), + error(What) + end. + +% mod_mam_utils:success_sql_query( +% Host, +% ["SELECT id, from_jid, message " +% "FROM mam_message ", +% Filter, Order, LimitSQL, Offset]). + +lookup_sql_binary(QueryType, Filters, Order) -> + iolist_to_binary(lookup_sql(QueryType, Filters, Order)). + +lookup_sql(QueryType, Filters, Order) -> + LimitSQL = limit_sql(QueryType), + OrderSQL = order_to_sql(Order), + FilterSQL = filters_to_sql(Filters), + ["SELECT ", columns_sql(QueryType), " " + "FROM mam_message ", + FilterSQL, OrderSQL, LimitSQL]. + +columns_sql(lookup) -> "id, from_jid, message"; +columns_sql(count) -> "COUNT(*)". + +limit_sql(lookup) -> rdbms_queries:limit_offset_sql(); +limit_sql(count) -> "". + +filters_to_columns(Filters) -> + [Column || {_Op, Column, _Value} <- Filters]. + +filters_to_args(Filters) -> + [Value || {_Op, _Column, Value} <- Filters]. + + +filters_to_statement_name(QueryType, Filters, Order) -> + QueryId = query_type_to_id(QueryType), + Ids = [type_to_id(Type) ++ column_to_id(Col) || {Type, Col, _Val} <- Filters], + OrderId = order_type_to_id(Order), + list_to_atom("mam_" ++ QueryId ++ "_" ++ OrderId ++ "_" ++ lists:append(Ids)). + +query_type_to_id(lookup) -> "lookup"; +query_type_to_id(count) -> "count". + +order_type_to_id(desc) -> "d"; +order_type_to_id(asc) -> "a"; +order_type_to_id(unordered) -> "u". + +order_to_sql(asc) -> " ORDER BY id "; +order_to_sql(desc) -> " ORDER BY id DESC "; +order_to_sql(unordered) -> " ". + +type_to_id(le) -> "le"; %% lower or equal +type_to_id(ge) -> "ge"; %% greater or equal +type_to_id(equal) -> "eq"; +type_to_id(lower) -> "lt"; %% lower than +type_to_id(greater) -> "gt"; %% greater than +type_to_id(like) -> "lk"; +type_to_id(limit) -> "li"; +type_to_id(offset) -> "of". + +column_to_id(id) -> "i"; +column_to_id(user_id) -> "u"; +column_to_id(remote_bare_jid) -> "b"; +column_to_id(remote_resource) -> "r"; +column_to_id(search_body) -> "s"; +%% fictional columns +column_to_id(limit) -> "l"; +column_to_id(offset) -> "o". + +filters_to_sql(Filters) -> + SQLs = [filter_to_sql(Filter) || Filter <- Filters], + case skip_undefined(SQLs) of + [] -> + ""; + Defined -> + [" WHERE ", rdbms_queries:join(Defined, " AND ")] + end. + +skip_undefined(List) -> + [X || X <- List, X =/= undefined]. + +filter_to_sql({Op, Column, _Value}) -> + filter_to_sql(Op, atom_to_list(Column)). + +filter_to_sql(limit, _) -> + undefined; +filter_to_sql(offset, _) -> + undefined; +filter_to_sql(lower, Column) -> + Column ++ " < ?"; +filter_to_sql(greater, Column) -> + Column ++ " > ?"; +filter_to_sql(le, Column) -> + Column ++ " <= ?"; +filter_to_sql(ge, Column) -> + Column ++ " >= ?"; +filter_to_sql(equal, Column) -> + Column ++ " = ?"; +filter_to_sql(like, Column) -> + Column ++ " LIKE ?". extract_gdpr_messages(Host, ArchiveID) -> Filter = ["WHERE user_id=", use_escaped_integer(escape_user_id(ArchiveID))], @@ -528,16 +635,10 @@ extract_gdpr_messages(Host, ArchiveID) -> %% @end %% "SELECT COUNT(*) as "index" FROM mam_message WHERE id <= '", UID -spec calc_index(Host :: jid:server(), - Filter :: filter(), IndexHintSQL :: string(), - SUID :: escaped_message_id()) -> non_neg_integer(). -calc_index(Host, Filter, IndexHintSQL, SUID) -> - {selected, [{BIndex}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) FROM mam_message ", - IndexHintSQL, Filter, - " AND id <= ", use_escaped_integer(SUID)]), - mongoose_rdbms:result_to_integer(BIndex). + Filter :: filter(), + ID :: mod_mam:message_id()) -> non_neg_integer(). +calc_index(Host, Filter, ID) -> + calc_count(Host, [{le, id, ID}|Filter]). %% @doc Count of elements in RSet before the passed element. %% @@ -545,31 +646,20 @@ calc_index(Host, Filter, IndexHintSQL, SUID) -> %% @end %% "SELECT COUNT(*) as "count" FROM mam_message WHERE id < '", UID -spec calc_before(Host :: jid:server(), - Filter :: filter(), IndexHintSQL :: string(), SUID :: escaped_message_id()) -> - non_neg_integer(). -calc_before(Host, Filter, IndexHintSQL, SUID) -> - {selected, [{BIndex}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) FROM mam_message ", - IndexHintSQL, Filter, - " AND id < ", use_escaped_integer(SUID)]), - mongoose_rdbms:result_to_integer(BIndex). - + Filter :: filter(), + ID :: mod_mam:message_id()) -> non_neg_integer(). +calc_before(Host, Filter, ID) -> + calc_count(Host, [{lower, id, ID}|Filter]). %% @doc Get the total result set size. %% "SELECT COUNT(*) as "count" FROM mam_message WHERE " -spec calc_count(Host :: jid:server(), - Filter :: filter(), IndexHintSQL :: string()) -> non_neg_integer(). -calc_count(Host, Filter, IndexHintSQL) -> + Filter :: filter()) -> non_neg_integer(). +calc_count(Host, Filter) -> {selected, [{BCount}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) FROM mam_message ", - IndexHintSQL, Filter]), + do_lookup_query(count, Host, Filter, unordered), mongoose_rdbms:result_to_integer(BCount). - -spec prepare_filter(Host :: jid:server(), UserID :: mod_mam:archive_id(), UserJID :: jid:jid(), Borders :: mod_mam:borders(), Start :: mod_mam:unix_timestamp() | undefined, @@ -577,51 +667,50 @@ calc_count(Host, Filter, IndexHintSQL) -> SearchText :: binary() | undefined) -> filter(). prepare_filter(Host, UserID, UserJID, Borders, Start, End, WithJID, SearchText) -> - {SWithJID, SWithResource} = + {MinWithJID, WithResource} = case WithJID of undefined -> {undefined, undefined}; #jid{lresource = <<>>} -> - {minify_and_escape_bare_jid(Host, UserJID, WithJID), undefined}; + {minify_bare_jid(Host, UserJID, WithJID), undefined}; #jid{lresource = WithLResource} -> - {minify_and_escape_bare_jid(Host, UserJID, WithJID), - mongoose_rdbms:escape_string(WithLResource)} + {minify_bare_jid(Host, UserJID, WithJID), + WithLResource} end, StartID = maybe_encode_compact_uuid(Start, 0), EndID = maybe_encode_compact_uuid(End, 255), StartID2 = apply_start_border(Borders, StartID), EndID2 = apply_end_border(Borders, EndID), - prepare_filter_sql(UserID, StartID2, EndID2, SWithJID, SWithResource, SearchText). - - --spec prepare_filter_sql(UserID :: non_neg_integer(), - StartID :: mod_mam:message_id() | undefined, - EndID :: mod_mam:message_id() | undefined, - SWithJID :: escaped_jid() | undefined, - SWithResource :: escaped_resource() | undefined, - SearchText :: binary() | undefined) -> filter(). -prepare_filter_sql(UserID, StartID, EndID, SWithJID, SWithResource, SearchText) -> - ["WHERE user_id=", use_escaped_integer(escape_user_id(UserID)), - case StartID of - undefined -> ""; - _ -> [" AND id >= ", use_escaped_integer(escape_integer(StartID))] - end, - case EndID of - undefined -> ""; - _ -> [" AND id <= ", use_escaped_integer(escape_integer(EndID))] - end, - case SWithJID of - undefined -> ""; - _ -> [" AND remote_bare_jid = ", use_escaped_string(SWithJID)] - end, - case SWithResource of - undefined -> ""; - _ -> [" AND remote_resource = ", use_escaped_string(SWithResource)] - end, - case SearchText of - undefined -> ""; - _ -> prepare_search_filters(SearchText) - end - ]. + prepare_filters(UserID, StartID2, EndID2, MinWithJID, WithResource, SearchText). + + +-spec prepare_filters(UserID :: non_neg_integer(), + StartID :: mod_mam:message_id() | undefined, + EndID :: mod_mam:message_id() | undefined, + WithJID :: binary() | undefined, + WithResource :: binary() | undefined, + SearchText :: binary() | undefined) -> filter(). +prepare_filters(UserID, StartID, EndID, WithJID, WithResource, SearchText) -> + [{equal, user_id, UserID}] ++ + case StartID of + undefined -> []; + _ -> [{ge, id, StartID}] + end ++ + case EndID of + undefined -> []; + _ -> [{le, id, EndID}] + end ++ + case WithJID of + undefined -> []; + _ -> [{equal, remote_bare_jid, WithJID}] + end ++ + case WithResource of + undefined -> []; + _ -> [{equal, remote_resource, WithResource}] + end ++ + case SearchText of + undefined -> []; + _ -> prepare_search_filters(SearchText) + end. %% Constructs a separate LIKE filter for each word. %% SearchText example is "word1%word2%word3". @@ -632,9 +721,8 @@ prepare_search_filters(SearchText) -> -spec prepare_search_filter(binary()) -> filter(). prepare_search_filter(Word) -> - [" AND search_body like ", %% Search for "%Word%" - mongoose_rdbms:use_escaped_like(mongoose_rdbms:escape_like(Word))]. + {like, search_body, <<"%", Word/binary, "%">>}. %% @doc #rsm_in{ %% max = non_neg_integer() | undefined, @@ -642,23 +730,21 @@ prepare_search_filter(Word) -> %% id = binary() | undefined, %% index = non_neg_integer() | undefined} -spec calc_offset(Host :: jid:server(), - Filter :: filter(), IndexHintSQL :: string(), PageSize :: non_neg_integer(), + Filter :: filter(), PageSize :: non_neg_integer(), TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in()) -> non_neg_integer(). -calc_offset(_LS, _F, _IH, _PS, _TC, #rsm_in{direction = undefined, index = Index}) +calc_offset(_LS, _F, _PS, _TC, #rsm_in{direction = undefined, index = Index}) when is_integer(Index) -> Index; %% Requesting the Last Page in a Result Set -calc_offset(_LS, _F, _IH, PS, TC, #rsm_in{direction = before, id = undefined}) -> +calc_offset(_LS, _F, PS, TC, #rsm_in{direction = before, id = undefined}) -> max(0, TC - PS); -calc_offset(Host, F, IH, PS, _TC, #rsm_in{direction = before, id = ID}) +calc_offset(Host, F, PS, _TC, #rsm_in{direction = before, id = ID}) when is_integer(ID) -> - SID = escape_message_id(ID), - max(0, calc_before(Host, F, IH, SID) - PS); -calc_offset(Host, F, IH, _PS, _TC, #rsm_in{direction = aft, id = ID}) + max(0, calc_before(Host, F, ID) - PS); +calc_offset(Host, F, _PS, _TC, #rsm_in{direction = aft, id = ID}) when is_integer(ID) -> - SID = escape_message_id(ID), - calc_index(Host, F, IH, SID); -calc_offset(_LS, _F, _IH, _PS, _TC, _RSM) -> + calc_index(Host, F, ID); +calc_offset(_LS, _F, _PS, _TC, _RSM) -> 0. escape_message_id(MessID) when is_integer(MessID) -> diff --git a/src/rdbms/mongoose_rdbms_odbc.erl b/src/rdbms/mongoose_rdbms_odbc.erl index 8f12e52f38..764cde654e 100644 --- a/src/rdbms/mongoose_rdbms_odbc.erl +++ b/src/rdbms/mongoose_rdbms_odbc.erl @@ -153,6 +153,8 @@ parse_row([], []) -> FieldName :: binary()) -> fun((term()) -> tuple()). field_name_to_mapper(_ServerType, _TableDesc, <<"limit">>) -> fun(P) -> {sql_integer, [P]} end; +field_name_to_mapper(_ServerType, _TableDesc, <<"offset">>) -> + fun(P) -> {sql_integer, [P]} end; field_name_to_mapper(_ServerType, TableDesc, FieldName) -> {_, ODBCType} = lists:keyfind(unicode:characters_to_list(FieldName), 1, TableDesc), case simple_type(just_type(ODBCType)) of diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index 998ae45e34..b3e1f38cea 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -33,6 +33,8 @@ get_db_specific_limits_binaries/1, get_db_specific_offset/2, add_limit_arg/2, + limit_offset_sql/0, + limit_offset_filters/2, sql_transaction/2, get_last/2, select_last/3, @@ -861,3 +863,19 @@ add_limit_arg(_, Limit, Args) -> get_db_specific_limits_binaries(Limit) -> {LimitSQL, LimitMSSQL} = get_db_specific_limits(Limit), {list_to_binary(LimitSQL), list_to_binary(LimitMSSQL)}. + +limit_offset_sql() -> + limit_offset_sql(?RDBMS_TYPE). + +limit_offset_sql(mssql) -> + <<" OFFSET (?) ROWS FETCH NEXT (?) ROWS ONLY">>; +limit_offset_sql(_) -> + <<" LIMIT ? OFFSET ?">>. + +limit_offset_filters(Limit, Offset) -> + limit_offset_filters(?RDBMS_TYPE, Limit, Offset). + +limit_offset_filters(mssql, Limit, Offset) -> + [{offset, offset, Offset}, {limit, limit, Limit}]; +limit_offset_filters(_, Limit, Offset) -> + [{limit, limit, Limit}, {offset, offset, Offset}]. From df87a87b17a5f3e37db1be4478806233122bf398 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 27 Nov 2020 10:24:43 +0100 Subject: [PATCH 03/92] Reorder function in mod_mam_rdbms_arch.erl --- src/mam/mod_mam_rdbms_arch.erl | 250 ++++++++++++++++----------------- 1 file changed, 123 insertions(+), 127 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 86b0391d8f..d9b3b5b1de 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -523,11 +523,129 @@ do_lookup_query(QueryType, Host, Filters, Order) -> error(What) end. -% mod_mam_utils:success_sql_query( -% Host, -% ["SELECT id, from_jid, message " -% "FROM mam_message ", -% Filter, Order, LimitSQL, Offset]). +%% @doc Calculate a zero-based index of the row with UID in the result test. +%% +%% If the element does not exists, the ID of the next element will +%% be returned instead. +%% @end +%% "SELECT COUNT(*) as "index" FROM mam_message WHERE id <= '", UID +-spec calc_index(Host :: jid:server(), + Filter :: filter(), + ID :: mod_mam:message_id()) -> non_neg_integer(). +calc_index(Host, Filter, ID) -> + calc_count(Host, [{le, id, ID}|Filter]). + +%% @doc Count of elements in RSet before the passed element. +%% +%% The element with the passed UID can be already deleted. +%% @end +%% "SELECT COUNT(*) as "count" FROM mam_message WHERE id < '", UID +-spec calc_before(Host :: jid:server(), + Filter :: filter(), + ID :: mod_mam:message_id()) -> non_neg_integer(). +calc_before(Host, Filter, ID) -> + calc_count(Host, [{lower, id, ID}|Filter]). + +%% @doc Get the total result set size. +%% "SELECT COUNT(*) as "count" FROM mam_message WHERE " +-spec calc_count(Host :: jid:server(), + Filter :: filter()) -> non_neg_integer(). +calc_count(Host, Filter) -> + {selected, [{BCount}]} = + do_lookup_query(count, Host, Filter, unordered), + mongoose_rdbms:result_to_integer(BCount). + +%% @doc #rsm_in{ +%% max = non_neg_integer() | undefined, +%% direction = before | aft | undefined, +%% id = binary() | undefined, +%% index = non_neg_integer() | undefined} +-spec calc_offset(Host :: jid:server(), + Filter :: filter(), PageSize :: non_neg_integer(), + TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in()) -> non_neg_integer(). +calc_offset(_LS, _F, _PS, _TC, #rsm_in{direction = undefined, index = Index}) + when is_integer(Index) -> + Index; +%% Requesting the Last Page in a Result Set +calc_offset(_LS, _F, PS, TC, #rsm_in{direction = before, id = undefined}) -> + max(0, TC - PS); +calc_offset(Host, F, PS, _TC, #rsm_in{direction = before, id = ID}) + when is_integer(ID) -> + max(0, calc_before(Host, F, ID) - PS); +calc_offset(Host, F, _PS, _TC, #rsm_in{direction = aft, id = ID}) + when is_integer(ID) -> + calc_index(Host, F, ID); +calc_offset(_LS, _F, _PS, _TC, _RSM) -> + 0. + +extract_gdpr_messages(Host, ArchiveID) -> + Filter = ["WHERE user_id=", use_escaped_integer(escape_user_id(ArchiveID))], + mod_mam_utils:success_sql_query( + Host, + ["SELECT id, from_jid, message " + "FROM mam_message ", + Filter, " ORDER BY id"]). + +escape_message_id(MessID) when is_integer(MessID) -> + escape_integer(MessID). + +escape_user_id(UserID) when is_integer(UserID) -> + escape_integer(UserID). + +%% @doc Strip resource, minify and escape JID. +minify_and_escape_bare_jid(Host, LocJID, JID) -> + escape_string(jid_to_stored_binary(Host, LocJID, jid:to_bare(JID))). + +minify_bare_jid(Host, LocJID, JID) -> + jid_to_stored_binary(Host, LocJID, jid:to_bare(JID)). + +maybe_encode_compact_uuid(undefined, _) -> + undefined; +maybe_encode_compact_uuid(Microseconds, NodeID) -> + encode_compact_uuid(Microseconds, NodeID). + +%% ---------------------------------------------------------------------- +%% Optimizations + +jid_to_stored_binary(Host, UserJID, JID) -> + Module = db_jid_codec(Host), + mam_jid:encode(Module, UserJID, JID). + +stored_binary_to_jid(Host, UserJID, BSrcJID) -> + Module = db_jid_codec(Host), + mam_jid:decode(Module, UserJID, BSrcJID). + +packet_to_stored_binary(Host, Packet) -> + Module = db_message_codec(Host), + mam_message:encode(Module, Packet). + +stored_binary_to_packet(Host, Bin) -> + Module = db_message_codec(Host), + mam_message:decode(Module, Bin). + +-spec db_jid_codec(jid:server()) -> module(). +db_jid_codec(Host) -> + gen_mod:get_module_opt(Host, ?MODULE, db_jid_format, mam_jid_mini). + +-spec db_message_codec(jid:server()) -> module(). +db_message_codec(Host) -> + gen_mod:get_module_opt(Host, ?MODULE, db_message_format, mam_message_compressed_eterm). + +%gdpr helpers +gdpr_decode_jid(Host, UserJID, BSrcJID) -> + Codec = mod_mam_meta:get_mam_module_opt(Host, ?MODULE, db_jid_format, mam_jid_mini), + JID = mam_jid:decode(Codec, UserJID, BSrcJID), + jid:to_binary(JID). + +gdpr_decode_packet(Host, SDataRaw) -> + Codec = mod_mam_meta:get_mam_module_opt(Host, ?MODULE, db_message_format, + mam_message_compressed_eterm), + Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), + Message = mam_message:decode(Codec, Data), + exml:to_binary(Message). + +%% ---------------------------------------------------------------------- +%% Prepared queries helpers lookup_sql_binary(QueryType, Filters, Order) -> iolist_to_binary(lookup_sql(QueryType, Filters, Order)). @@ -620,46 +738,6 @@ filter_to_sql(equal, Column) -> filter_to_sql(like, Column) -> Column ++ " LIKE ?". -extract_gdpr_messages(Host, ArchiveID) -> - Filter = ["WHERE user_id=", use_escaped_integer(escape_user_id(ArchiveID))], - mod_mam_utils:success_sql_query( - Host, - ["SELECT id, from_jid, message " - "FROM mam_message ", - Filter, " ORDER BY id"]). - -%% @doc Calculate a zero-based index of the row with UID in the result test. -%% -%% If the element does not exists, the ID of the next element will -%% be returned instead. -%% @end -%% "SELECT COUNT(*) as "index" FROM mam_message WHERE id <= '", UID --spec calc_index(Host :: jid:server(), - Filter :: filter(), - ID :: mod_mam:message_id()) -> non_neg_integer(). -calc_index(Host, Filter, ID) -> - calc_count(Host, [{le, id, ID}|Filter]). - -%% @doc Count of elements in RSet before the passed element. -%% -%% The element with the passed UID can be already deleted. -%% @end -%% "SELECT COUNT(*) as "count" FROM mam_message WHERE id < '", UID --spec calc_before(Host :: jid:server(), - Filter :: filter(), - ID :: mod_mam:message_id()) -> non_neg_integer(). -calc_before(Host, Filter, ID) -> - calc_count(Host, [{lower, id, ID}|Filter]). - -%% @doc Get the total result set size. -%% "SELECT COUNT(*) as "count" FROM mam_message WHERE " --spec calc_count(Host :: jid:server(), - Filter :: filter()) -> non_neg_integer(). -calc_count(Host, Filter) -> - {selected, [{BCount}]} = - do_lookup_query(count, Host, Filter, unordered), - mongoose_rdbms:result_to_integer(BCount). - -spec prepare_filter(Host :: jid:server(), UserID :: mod_mam:archive_id(), UserJID :: jid:jid(), Borders :: mod_mam:borders(), Start :: mod_mam:unix_timestamp() | undefined, @@ -682,7 +760,6 @@ prepare_filter(Host, UserID, UserJID, Borders, Start, End, WithJID, SearchText) EndID2 = apply_end_border(Borders, EndID), prepare_filters(UserID, StartID2, EndID2, MinWithJID, WithResource, SearchText). - -spec prepare_filters(UserID :: non_neg_integer(), StartID :: mod_mam:message_id() | undefined, EndID :: mod_mam:message_id() | undefined, @@ -723,84 +800,3 @@ prepare_search_filters(SearchText) -> prepare_search_filter(Word) -> %% Search for "%Word%" {like, search_body, <<"%", Word/binary, "%">>}. - -%% @doc #rsm_in{ -%% max = non_neg_integer() | undefined, -%% direction = before | aft | undefined, -%% id = binary() | undefined, -%% index = non_neg_integer() | undefined} --spec calc_offset(Host :: jid:server(), - Filter :: filter(), PageSize :: non_neg_integer(), - TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in()) -> non_neg_integer(). -calc_offset(_LS, _F, _PS, _TC, #rsm_in{direction = undefined, index = Index}) - when is_integer(Index) -> - Index; -%% Requesting the Last Page in a Result Set -calc_offset(_LS, _F, PS, TC, #rsm_in{direction = before, id = undefined}) -> - max(0, TC - PS); -calc_offset(Host, F, PS, _TC, #rsm_in{direction = before, id = ID}) - when is_integer(ID) -> - max(0, calc_before(Host, F, ID) - PS); -calc_offset(Host, F, _PS, _TC, #rsm_in{direction = aft, id = ID}) - when is_integer(ID) -> - calc_index(Host, F, ID); -calc_offset(_LS, _F, _PS, _TC, _RSM) -> - 0. - -escape_message_id(MessID) when is_integer(MessID) -> - escape_integer(MessID). - -escape_user_id(UserID) when is_integer(UserID) -> - escape_integer(UserID). - -%% @doc Strip resource, minify and escape JID. -minify_and_escape_bare_jid(Host, LocJID, JID) -> - escape_string(jid_to_stored_binary(Host, LocJID, jid:to_bare(JID))). - -minify_bare_jid(Host, LocJID, JID) -> - jid_to_stored_binary(Host, LocJID, jid:to_bare(JID)). - -maybe_encode_compact_uuid(undefined, _) -> - undefined; -maybe_encode_compact_uuid(Microseconds, NodeID) -> - encode_compact_uuid(Microseconds, NodeID). - -%% ---------------------------------------------------------------------- -%% Optimizations - -jid_to_stored_binary(Host, UserJID, JID) -> - Module = db_jid_codec(Host), - mam_jid:encode(Module, UserJID, JID). - -stored_binary_to_jid(Host, UserJID, BSrcJID) -> - Module = db_jid_codec(Host), - mam_jid:decode(Module, UserJID, BSrcJID). - -packet_to_stored_binary(Host, Packet) -> - Module = db_message_codec(Host), - mam_message:encode(Module, Packet). - -stored_binary_to_packet(Host, Bin) -> - Module = db_message_codec(Host), - mam_message:decode(Module, Bin). - --spec db_jid_codec(jid:server()) -> module(). -db_jid_codec(Host) -> - gen_mod:get_module_opt(Host, ?MODULE, db_jid_format, mam_jid_mini). - --spec db_message_codec(jid:server()) -> module(). -db_message_codec(Host) -> - gen_mod:get_module_opt(Host, ?MODULE, db_message_format, mam_message_compressed_eterm). - -%gdpr helpers -gdpr_decode_jid(Host, UserJID, BSrcJID) -> - Codec = mod_mam_meta:get_mam_module_opt(Host, ?MODULE, db_jid_format, mam_jid_mini), - JID = mam_jid:decode(Codec, UserJID, BSrcJID), - jid:to_binary(JID). - -gdpr_decode_packet(Host, SDataRaw) -> - Codec = mod_mam_meta:get_mam_module_opt(Host, ?MODULE, db_message_format, - mam_message_compressed_eterm), - Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Message = mam_message:decode(Codec, Data), - exml:to_binary(Message). From b2710e06de1aaa80dd54b7160152f61b46de66ae Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 27 Nov 2020 10:33:01 +0100 Subject: [PATCH 04/92] Convert gdpr query to prepared --- src/mam/mod_mam_rdbms_arch.erl | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index d9b3b5b1de..b23d9e8e6e 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -91,6 +91,13 @@ start(Host, Opts) -> " WHERE user_id = ? AND remote_bare_jid = ? " " AND origin_id = ? AND direction = ?" " ORDER BY id DESC ", LimitSQL/binary>>]), + + mongoose_rdbms:prepare(mam_extract_gdpr_messages, mam_message, + [user_id], + [<<"SELECT id, from_jid, message " + " FROM mam_message " + " WHERE user_id=? " + " ORDER BY id">>]), ok. @@ -356,7 +363,6 @@ lookup_messages_opt_count(Host, UserJID, {ok, {MessageRowsCount, 0, rows_to_uniform_format(Host, UserJID, MessageRows)}}; false -> - IndexHintSQL = index_hint_sql(Host), FirstID = row_to_message_id(hd(MessageRows)), Offset = calc_count(Host, before_id(FirstID, Filter)), {ok, {Offset + MessageRowsCount, Offset, @@ -373,7 +379,6 @@ lookup_messages_opt_count(Host, UserJID, {ok, {Offset + MessageRowsCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}}; false -> - IndexHintSQL = index_hint_sql(Host), LastID = row_to_message_id(lists:last(MessageRows)), CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), {ok, {Offset + MessageRowsCount + CountAfterLastID, Offset, @@ -390,7 +395,6 @@ lookup_messages_opt_count(Host, UserJID, {ok, {MessageRowsCount, 0, rows_to_uniform_format(Host, UserJID, MessageRows)}}; false -> - IndexHintSQL = index_hint_sql(Host), LastID = row_to_message_id(lists:last(MessageRows)), CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), {ok, {MessageRowsCount + CountAfterLastID, 0, @@ -579,12 +583,7 @@ calc_offset(_LS, _F, _PS, _TC, _RSM) -> 0. extract_gdpr_messages(Host, ArchiveID) -> - Filter = ["WHERE user_id=", use_escaped_integer(escape_user_id(ArchiveID))], - mod_mam_utils:success_sql_query( - Host, - ["SELECT id, from_jid, message " - "FROM mam_message ", - Filter, " ORDER BY id"]). + mod_mam_utils:success_sql_execute(Host, mam_extract_gdpr_messages, [ArchiveID]). escape_message_id(MessID) when is_integer(MessID) -> escape_integer(MessID). From 3e333cf39bd3fddc27b1c3af4a8e802e91f05f92 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 27 Nov 2020 10:54:00 +0100 Subject: [PATCH 05/92] ADd maybe_reserve helper --- src/mam/mod_mam_rdbms_arch.erl | 41 ++++++++++++++++------------------ 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index b23d9e8e6e..d0d0047c63 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -332,21 +332,21 @@ lookup_messages_simple(Host, UserJID, #rsm_in{direction = aft, id = ID}, PageSize, Filter) -> %% Get last rows from result set - MessageRows = extract_messages(Host, after_id(ID, Filter), 0, PageSize, false), + MessageRows = extract_messages(Host, after_id(ID, Filter), 0, PageSize, asc), {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}; lookup_messages_simple(Host, UserJID, #rsm_in{direction = before, id = ID}, PageSize, Filter) -> - MessageRows = extract_messages(Host, before_id(ID, Filter), 0, PageSize, true), + MessageRows = extract_messages(Host, before_id(ID, Filter), 0, PageSize, desc), {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}; lookup_messages_simple(Host, UserJID, #rsm_in{direction = undefined, index = Offset}, PageSize, Filter) -> %% Apply offset - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), + MessageRows = extract_messages(Host, Filter, Offset, PageSize, asc), {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}; lookup_messages_simple(Host, UserJID, undefined, PageSize, Filter) -> - MessageRows = extract_messages(Host, Filter, 0, PageSize, false), + MessageRows = extract_messages(Host, Filter, 0, PageSize, asc), {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}. %% Cases that cannot be optimized and used with this function: @@ -356,7 +356,7 @@ lookup_messages_opt_count(Host, UserJID, #rsm_in{direction = before, id = undefined}, PageSize, Filter) -> %% Last page - MessageRows = extract_messages(Host, Filter, 0, PageSize, true), + MessageRows = extract_messages(Host, Filter, 0, PageSize, desc), MessageRowsCount = length(MessageRows), case MessageRowsCount < PageSize of true -> @@ -372,7 +372,7 @@ lookup_messages_opt_count(Host, UserJID, #rsm_in{direction = undefined, index = Offset}, PageSize, Filter) -> %% By offset - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), + MessageRows = extract_messages(Host, Filter, Offset, PageSize, asc), MessageRowsCount = length(MessageRows), case MessageRowsCount < PageSize of true -> @@ -388,7 +388,7 @@ lookup_messages_opt_count(Host, UserJID, undefined, PageSize, Filter) -> %% First page - MessageRows = extract_messages(Host, Filter, 0, PageSize, false), + MessageRows = extract_messages(Host, Filter, 0, PageSize, asc), MessageRowsCount = length(MessageRows), case MessageRowsCount < PageSize of true -> @@ -406,7 +406,7 @@ lookup_messages_regular(Host, UserJID, PageSize, Filter) when ID =/= undefined -> TotalCount = calc_count(Host, Filter), Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, from_id(ID, Filter), 0, PageSize + 1, false), + MessageRows = extract_messages(Host, from_id(ID, Filter), 0, PageSize + 1, asc), Result = {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}, mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); lookup_messages_regular(Host, UserJID, @@ -414,14 +414,14 @@ lookup_messages_regular(Host, UserJID, PageSize, Filter) when ID =/= undefined -> TotalCount = calc_count(Host, Filter), Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, to_id(ID, Filter), 0, PageSize + 1, true), + MessageRows = extract_messages(Host, to_id(ID, Filter), 0, PageSize + 1, desc), Result = {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}, mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); lookup_messages_regular(Host, UserJID, RSM, PageSize, Filter) -> TotalCount = calc_count(Host, Filter), Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), + MessageRows = extract_messages(Host, Filter, Offset, PageSize, asc), {ok, {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}}. -spec after_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). @@ -475,23 +475,21 @@ remove_archive(Host, UserID) -> -type msg() :: {binary(), jid:literal_jid(), binary()}. -spec extract_messages(Host :: jid:server(), Filter :: filter(), IOffset :: non_neg_integer(), IMax :: pos_integer(), - ReverseLimit :: boolean()) -> [msg()]. + Order :: asc | desc) -> [msg()]. extract_messages(_Host, _Filter, _IOffset, 0, _) -> []; -extract_messages(Host, Filter, IOffset, IMax, false) -> +extract_messages(Host, Filter, IOffset, IMax, Order) -> {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, asc), + do_extract_messages(Host, Filter, IOffset, IMax, Order), ?LOG_DEBUG(#{what => mam_extract_messages, mam_filter => Filter, offset => IOffset, max => IMax, host => Host, message_rows => MessageRows}), - MessageRows; -extract_messages(Host, Filter, IOffset, IMax, true) -> - {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, desc), - ?LOG_DEBUG(#{what => mam_extract_messages, - mam_filter => Filter, offset => IOffset, max => IMax, - host => Host, message_rows => MessageRows}), - lists:reverse(MessageRows). + maybe_reserve(Order, MessageRows). + +maybe_reserve(asc, List) -> + List; +maybe_reserve(desc, List) -> + lists:reverse(List). do_extract_messages(Host, Filters, IOffset, IMax, Order) -> Filters2 = Filters ++ rdbms_queries:limit_offset_filters(IMax, IOffset), @@ -669,7 +667,6 @@ filters_to_columns(Filters) -> filters_to_args(Filters) -> [Value || {_Op, _Column, Value} <- Filters]. - filters_to_statement_name(QueryType, Filters, Order) -> QueryId = query_type_to_id(QueryType), Ids = [type_to_id(Type) ++ column_to_id(Col) || {Type, Col, _Val} <- Filters], From d7b469d6a28d66b78061fc5a65984e87bb7b45a5 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 27 Nov 2020 11:55:20 +0100 Subject: [PATCH 06/92] Compactify code for do_lookup_messages --- src/mam/mod_mam_rdbms_arch.erl | 177 ++++++++++++--------------------- src/mam/mod_mam_utils.erl | 10 +- 2 files changed, 70 insertions(+), 117 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index d0d0047c63..fcc0c19e0a 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -42,12 +42,6 @@ [apply_start_border/2, apply_end_border/2]). --import(mongoose_rdbms, - [escape_string/1, - escape_integer/1, - use_escaped_string/1, - use_escaped_integer/1]). - -include("mongoose.hrl"). -include("jlib.hrl"). -include_lib("exml/include/exml.hrl"). @@ -58,10 +52,6 @@ -type filter_field() :: tuple(). -type filter() :: [filter_field()]. --type escaped_message_id() :: mongoose_rdbms:escape_string(). --type escaped_jid() :: mongoose_rdbms:escaped_string(). --type escaped_resource() :: mongoose_rdbms:escaped_string(). - %% ---------------------------------------------------------------------- %% gen_mod callbacks @@ -265,24 +255,13 @@ lookup_messages({error, _Reason}=Result, _Host, _Params) -> Result; lookup_messages(_Result, Host, Params) -> try - UserID = maps:get(archive_id, Params), - UserJID = maps:get(owner_jid, Params), RSM = maps:get(rsm, Params), - Borders = maps:get(borders, Params), - Start = maps:get(start_ts, Params), - End = maps:get(end_ts, Params), - Now = maps:get(now, Params), - WithJID = maps:get(with_jid, Params), SearchText = maps:get(search_text, Params), - PageSize = maps:get(page_size, Params), - IsSimple = maps:get(is_simple, Params), - - do_lookup_messages(Host, - UserID, UserJID, RSM, Borders, - Start, End, Now, WithJID, - mod_mam_utils:normalize_search_text(SearchText), - PageSize, - IsSimple, is_opt_count_supported_for(RSM)) + Params2 = Params#{ + norm_search_text => mod_mam_utils:normalize_search_text(SearchText), + opt_count_supported => is_opt_count_supported_for(RSM)}, + Filter = prepare_filter(Host, Params2), + do_lookup_messages(Host, Filter, Params2) catch _Type:Reason:S -> {error, {Reason, {stacktrace, S}}} end. @@ -304,51 +283,40 @@ is_opt_count_supported_for(_) -> %% - we can reduce number of queries if we skip counting for small data sets; %% - sometimes we want not to count at all %% (for example, our client side counts ones and keep the information) -do_lookup_messages(Host, UserID, UserJID, - RSM, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, true, _) -> - %% Simple query without calculating offset and total count - Filter = prepare_filter(Host, UserID, UserJID, Borders, Start, End, WithJID, SearchText), - lookup_messages_simple(Host, UserJID, RSM, PageSize, Filter); -do_lookup_messages(Host, UserID, UserJID, - RSM, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, opt_count, true) -> - %% Extract messages first than calculate offset and total count - %% Useful for small result sets (less than one page, than one query is enough) - Filter = prepare_filter(Host, UserID, UserJID, Borders, Start, End, WithJID, SearchText), - lookup_messages_opt_count(Host, UserJID, RSM, PageSize, Filter); -do_lookup_messages(Host, UserID, UserJID, - RSM, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, _, _) -> - %% Unsupported opt_count or just a regular query - %% Calculate offset and total count first than extract messages - Filter = prepare_filter(Host, UserID, UserJID, Borders, Start, End, WithJID, SearchText), - lookup_messages_regular(Host, UserJID, RSM, PageSize, Filter). - -lookup_messages_simple(Host, UserJID, - #rsm_in{direction = aft, id = ID}, - PageSize, Filter) -> - %% Get last rows from result set - MessageRows = extract_messages(Host, after_id(ID, Filter), 0, PageSize, asc), - {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}; -lookup_messages_simple(Host, UserJID, - #rsm_in{direction = before, id = ID}, - PageSize, Filter) -> - MessageRows = extract_messages(Host, before_id(ID, Filter), 0, PageSize, desc), - {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}; -lookup_messages_simple(Host, UserJID, - #rsm_in{direction = undefined, index = Offset}, - PageSize, Filter) -> - %% Apply offset - MessageRows = extract_messages(Host, Filter, Offset, PageSize, asc), - {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}; -lookup_messages_simple(Host, UserJID, undefined, PageSize, Filter) -> - MessageRows = extract_messages(Host, Filter, 0, PageSize, asc), +do_lookup_messages(Host, Filter, Params = #{owner_jid := UserJID, rsm := RSM, + page_size := PageSize}) -> + case Params of + #{is_simple := true} -> + %% Simple query without calculating offset and total count + lookup_messages_simple(Host, UserJID, RSM, PageSize, Filter); + #{is_simple := opt_count, opt_count_supported := true} -> + %% Extract messages first than calculate offset and total count + %% Useful for small result sets (less than one page, than one query is enough) + lookup_messages_opt_count(Host, UserJID, RSM, PageSize, Filter); + _ -> + %% Unsupported opt_count or just a regular query + %% Calculate offset and total count first than extract messages + lookup_messages_regular(Host, UserJID, RSM, PageSize, Filter) + end. + +lookup_messages_simple(Host, UserJID, RSM, PageSize, Filter) -> + {Filter2, Offset, Order} = rsm_to_filter(RSM, Filter), + MessageRows = extract_messages(Host, Filter2, Offset, PageSize, Order), {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}. +rsm_to_filter(RSM, Filter) -> + case RSM of + %% Get last rows from result set + #rsm_in{direction = aft, id = ID} -> + {after_id(ID, Filter), 0, asc}; + #rsm_in{direction = before, id = ID} -> + {before_id(ID, Filter), 0, desc}; + #rsm_in{direction = undefined, index = Index} -> + {Filter, Index, asc}; + undefined -> + {Filter, 0, asc} + end. + %% Cases that cannot be optimized and used with this function: %% - #rsm_in{direction = aft, id = ID} %% - #rsm_in{direction = before, id = ID} @@ -401,45 +369,37 @@ lookup_messages_opt_count(Host, UserJID, rows_to_uniform_format(Host, UserJID, MessageRows)}} end. -lookup_messages_regular(Host, UserJID, - RSM = #rsm_in{direction = aft, id = ID}, - PageSize, Filter) when ID =/= undefined -> +lookup_messages_regular(Host, UserJID, RSM, PageSize, Filter) -> TotalCount = calc_count(Host, Filter), Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, from_id(ID, Filter), 0, PageSize + 1, asc), - Result = {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}, - mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); -lookup_messages_regular(Host, UserJID, - RSM = #rsm_in{direction = before, id = ID}, - PageSize, Filter) when ID =/= undefined -> - TotalCount = calc_count(Host, Filter), - Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, to_id(ID, Filter), 0, PageSize + 1, desc), + MessageRows = + case RSM of + #rsm_in{direction = aft, id = ID} -> + extract_messages(Host, from_id(ID, Filter), 0, PageSize + 1, asc); + #rsm_in{direction = before, id = ID} when ID =/= undefined -> + extract_messages(Host, to_id(ID, Filter), 0, PageSize + 1, desc); + _ -> + extract_messages(Host, Filter, Offset, PageSize, asc) + end, Result = {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}, - mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); -lookup_messages_regular(Host, UserJID, RSM, - PageSize, Filter) -> - TotalCount = calc_count(Host, Filter), - Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, Filter, Offset, PageSize, asc), - {ok, {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}}. + mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result). --spec after_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). +-spec after_id(ID :: mod_mam:message_id(), Filter :: filter()) -> filter(). after_id(ID, Filter) -> [{greater, id, ID}|Filter]. --spec before_id(ID :: escaped_message_id() | undefined, +-spec before_id(ID :: mod_mam:message_id() | undefined, Filter :: filter()) -> filter(). before_id(undefined, Filter) -> Filter; before_id(ID, Filter) -> [{lower, id, ID}|Filter]. --spec from_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). +-spec from_id(ID :: mod_mam:message_id(), Filter :: filter()) -> filter(). from_id(ID, Filter) -> [{ge, id, ID}|Filter]. --spec to_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). +-spec to_id(ID :: mod_mam:message_id(), Filter :: filter()) -> filter(). to_id(ID, Filter) -> [{le, id, ID}|Filter]. @@ -486,11 +446,6 @@ extract_messages(Host, Filter, IOffset, IMax, Order) -> host => Host, message_rows => MessageRows}), maybe_reserve(Order, MessageRows). -maybe_reserve(asc, List) -> - List; -maybe_reserve(desc, List) -> - lists:reverse(List). - do_extract_messages(Host, Filters, IOffset, IMax, Order) -> Filters2 = Filters ++ rdbms_queries:limit_offset_filters(IMax, IOffset), do_lookup_query(lookup, Host, Filters2, Order). @@ -580,19 +535,14 @@ calc_offset(Host, F, _PS, _TC, #rsm_in{direction = aft, id = ID}) calc_offset(_LS, _F, _PS, _TC, _RSM) -> 0. +maybe_reserve(asc, List) -> + List; +maybe_reserve(desc, List) -> + lists:reverse(List). + extract_gdpr_messages(Host, ArchiveID) -> mod_mam_utils:success_sql_execute(Host, mam_extract_gdpr_messages, [ArchiveID]). -escape_message_id(MessID) when is_integer(MessID) -> - escape_integer(MessID). - -escape_user_id(UserID) when is_integer(UserID) -> - escape_integer(UserID). - -%% @doc Strip resource, minify and escape JID. -minify_and_escape_bare_jid(Host, LocJID, JID) -> - escape_string(jid_to_stored_binary(Host, LocJID, jid:to_bare(JID))). - minify_bare_jid(Host, LocJID, JID) -> jid_to_stored_binary(Host, LocJID, jid:to_bare(JID)). @@ -734,13 +684,12 @@ filter_to_sql(equal, Column) -> filter_to_sql(like, Column) -> Column ++ " LIKE ?". --spec prepare_filter(Host :: jid:server(), UserID :: mod_mam:archive_id(), - UserJID :: jid:jid(), Borders :: mod_mam:borders(), - Start :: mod_mam:unix_timestamp() | undefined, - End :: mod_mam:unix_timestamp() | undefined, WithJID :: jid:jid(), - SearchText :: binary() | undefined) - -> filter(). -prepare_filter(Host, UserID, UserJID, Borders, Start, End, WithJID, SearchText) -> +-spec prepare_filter(Host :: jid:server(), Params :: map()) -> filter(). +prepare_filter(Host, #{ + archive_id := UserID, owner_jid := UserJID, + borders := Borders, start_ts := Start, end_ts := End, + with_jid := WithJID, norm_search_text := SearchText + }) -> {MinWithJID, WithResource} = case WithJID of undefined -> {undefined, undefined}; diff --git a/src/mam/mod_mam_utils.erl b/src/mam/mod_mam_utils.erl index 14a2b62545..1372611fbd 100644 --- a/src/mam/mod_mam_utils.erl +++ b/src/mam/mod_mam_utils.erl @@ -1144,12 +1144,16 @@ is_policy_violation(TotalCount, Offset, MaxResultLimit, LimitPassed) -> %% return (up to) PageSize messages. %% @end -spec check_for_item_not_found(RSM, PageSize, LookupResult) -> R when - RSM :: jlib:rsm_in(), + RSM :: jlib:rsm_in() | undefined, PageSize :: non_neg_integer(), LookupResult :: mod_mam:lookup_result(), R :: {ok, mod_mam:lookup_result()} | {error, item_not_found}. +check_for_item_not_found(undefined, _PageSize, Result) -> + {ok, Result}; +check_for_item_not_found(#rsm_in{id = undefined}, _PageSize, Result) -> + {ok, Result}; check_for_item_not_found(#rsm_in{direction = before, id = ID}, - PageSize, {TotalCount, Offset, MessageRows}) when ID =/= undefined -> + PageSize, {TotalCount, Offset, MessageRows}) -> case maybe_last(MessageRows) of {ok, {ID, _, _}} = _IntervalEndpoint -> Page = lists:sublist(MessageRows, PageSize), @@ -1158,7 +1162,7 @@ check_for_item_not_found(#rsm_in{direction = before, id = ID}, {error, item_not_found} end; check_for_item_not_found(#rsm_in{direction = aft, id = ID}, - _PageSize, {TotalCount, Offset, MessageRows0}) when ID =/= undefined -> + _PageSize, {TotalCount, Offset, MessageRows0}) -> case MessageRows0 of [{ID, _, _} = _IntervalEndpoint | MessageRows] -> {ok, {TotalCount, Offset, MessageRows}}; From a6101aca9c4062777cd539d2bdcfc8dd836a18fb Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 10:37:21 +0100 Subject: [PATCH 07/92] Adress review commits --- src/mam/mod_mam_rdbms_arch.erl | 31 +++++++++++++++++++------------ src/rdbms/mongoose_rdbms.erl | 6 +++++- src/rdbms/rdbms_queries.erl | 8 ++++++-- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index fcc0c19e0a..f492783a2c 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -50,8 +50,17 @@ %% ---------------------------------------------------------------------- %% Types --type filter_field() :: tuple(). --type filter() :: [filter_field()]. +-type column() :: atom(). +-type filter_field() :: {like, column(), binary()} + | {le, column(), integer()} + | {ge, column(), integer()} + | {equal, column(), integer() | binary()} + | {lower, column(), integer()} + | {greater, column(), integer()} + | {limit, limit, integer()} + | {offset, offset, integer()}. + +-type filter() :: [filter_field()]. %% ---------------------------------------------------------------------- %% gen_mod callbacks @@ -148,8 +157,8 @@ encode_direction(outgoing) -> <<"O">>. -spec archive_size(Size :: integer(), Host :: jid:server(), ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). archive_size(Size, Host, UserID, _UserJID) when is_integer(Size) -> - {selected, [{BSize}]} = mod_mam_utils:success_sql_execute(Host, mam_archive_size, [UserID]), - mongoose_rdbms:result_to_integer(BSize). + Result = mod_mam_utils:success_sql_execute(Host, mam_archive_size, [UserID]), + mongoose_rdbms:selected_to_integer(Result). -spec index_hint_sql(jid:server()) -> string(). @@ -185,10 +194,8 @@ retract_message(Host, #{archive_id := UserID, direction := Dir, packet := Packet}) -> case mod_mam_utils:get_retract_id(mod_mam, Host, Packet) of - none -> - ok; - OriginIDToRetract -> - retract_message(Host, UserID, LocJID, RemJID, OriginIDToRetract, Dir) + none -> ok; + OriginIDToRetract -> retract_message(Host, UserID, LocJID, RemJID, OriginIDToRetract, Dir) end. retract_message(Host, UserID, LocJID, RemJID, OriginID, Dir) -> @@ -444,7 +451,7 @@ extract_messages(Host, Filter, IOffset, IMax, Order) -> ?LOG_DEBUG(#{what => mam_extract_messages, mam_filter => Filter, offset => IOffset, max => IMax, host => Host, message_rows => MessageRows}), - maybe_reserve(Order, MessageRows). + maybe_reverse(Order, MessageRows). do_extract_messages(Host, Filters, IOffset, IMax, Order) -> Filters2 = Filters ++ rdbms_queries:limit_offset_filters(IMax, IOffset), @@ -535,9 +542,9 @@ calc_offset(Host, F, _PS, _TC, #rsm_in{direction = aft, id = ID}) calc_offset(_LS, _F, _PS, _TC, _RSM) -> 0. -maybe_reserve(asc, List) -> +maybe_reverse(asc, List) -> List; -maybe_reserve(desc, List) -> +maybe_reverse(desc, List) -> lists:reverse(List). extract_gdpr_messages(Host, ArchiveID) -> @@ -741,7 +748,7 @@ prepare_search_filters(SearchText) -> Words = binary:split(SearchText, <<"%">>, [global]), [prepare_search_filter(Word) || Word <- Words]. --spec prepare_search_filter(binary()) -> filter(). +-spec prepare_search_filter(binary()) -> filter_field(). prepare_search_filter(Word) -> %% Search for "%Word%" {like, search_body, <<"%", Word/binary, "%">>}. diff --git a/src/rdbms/mongoose_rdbms.erl b/src/rdbms/mongoose_rdbms.erl index f92c3ca669..300e77a477 100644 --- a/src/rdbms/mongoose_rdbms.erl +++ b/src/rdbms/mongoose_rdbms.erl @@ -115,7 +115,8 @@ use_escaped_null/1]). %% count / integra types decoding --export([result_to_integer/1]). +-export([result_to_integer/1, + selected_to_integer/1]). %% gen_server callbacks -export([init/1, @@ -412,6 +413,9 @@ result_to_integer(Int) when is_integer(Int) -> result_to_integer(Bin) when is_binary(Bin) -> binary_to_integer(Bin). +selected_to_integer({selected, [{BInt}]}) -> + result_to_integer(BInt). + %% pgsql returns booleans as "t" or "f" -spec to_bool(binary() | string() | atom() | integer() | any()) -> boolean(). to_bool(B) when is_binary(B) -> diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index b3e1f38cea..c50340dfb3 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -835,9 +835,13 @@ get_db_specific_offset(Offset, Limit) -> do_get_db_specific_offset(?RDBMS_TYPE, integer_to_list(Offset), integer_to_list(Limit)). -do_get_db_specific_limits(mssql, LimitStr, false) -> +%% Arguments: +%% - Type (atom) - database type +%% - LimitStr (string) - a field value +%% - Wrap (boolean) - add parentheses around a field for MSSQL +do_get_db_specific_limits(mssql, LimitStr, _Wrap = false) -> {"", "TOP " ++ LimitStr}; -do_get_db_specific_limits(mssql, LimitStr, true) -> +do_get_db_specific_limits(mssql, LimitStr, _Wrap = true) -> {"", "TOP (" ++ LimitStr ++ ")"}; do_get_db_specific_limits(_, LimitStr, _Wrap) -> {"LIMIT " ++ LimitStr, ""}. From 43d43efa9fe7cceacd4d4470259b007f85b8cc00 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 13:33:23 +0100 Subject: [PATCH 08/92] Use lookup_field record in mod_mam_rdbms_arch --- src/mam/mod_mam_rdbms_arch.erl | 125 +++++++++++++++++---------------- 1 file changed, 65 insertions(+), 60 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index f492783a2c..a7b64ba00a 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -6,6 +6,8 @@ %%%------------------------------------------------------------------- -module(mod_mam_rdbms_arch). +-define(SEARCH_WORDS_LIMIT, 10). + %% ---------------------------------------------------------------------- %% Exports @@ -263,12 +265,10 @@ lookup_messages({error, _Reason}=Result, _Host, _Params) -> lookup_messages(_Result, Host, Params) -> try RSM = maps:get(rsm, Params), - SearchText = maps:get(search_text, Params), - Params2 = Params#{ - norm_search_text => mod_mam_utils:normalize_search_text(SearchText), - opt_count_supported => is_opt_count_supported_for(RSM)}, - Filter = prepare_filter(Host, Params2), - do_lookup_messages(Host, Filter, Params2) + Params2 = Params#{opt_count_supported => is_opt_count_supported_for(RSM)}, + Params3 = extend_params(Host, Params2), + Filter = prepare_filter(Params3), + do_lookup_messages(Host, Filter, Params3) catch _Type:Reason:S -> {error, {Reason, {stacktrace, S}}} end. @@ -550,6 +550,11 @@ maybe_reverse(desc, List) -> extract_gdpr_messages(Host, ArchiveID) -> mod_mam_utils:success_sql_execute(Host, mam_extract_gdpr_messages, [ArchiveID]). +maybe_minify_bare_jid(_Host, _LocJID, undefined) -> + undefined; +maybe_minify_bare_jid(Host, LocJID, JID) -> + minify_bare_jid(Host, LocJID, JID). + minify_bare_jid(Host, LocJID, JID) -> jid_to_stored_binary(Host, LocJID, jid:to_bare(JID)). @@ -691,64 +696,64 @@ filter_to_sql(equal, Column) -> filter_to_sql(like, Column) -> Column ++ " LIKE ?". --spec prepare_filter(Host :: jid:server(), Params :: map()) -> filter(). -prepare_filter(Host, #{ - archive_id := UserID, owner_jid := UserJID, - borders := Borders, start_ts := Start, end_ts := End, - with_jid := WithJID, norm_search_text := SearchText - }) -> - {MinWithJID, WithResource} = - case WithJID of - undefined -> {undefined, undefined}; - #jid{lresource = <<>>} -> - {minify_bare_jid(Host, UserJID, WithJID), undefined}; - #jid{lresource = WithLResource} -> - {minify_bare_jid(Host, UserJID, WithJID), - WithLResource} - end, +extend_params(Host, #{owner_jid := UserJID, borders := Borders, + start_ts := Start, end_ts := End, with_jid := WithJID, + search_text := SearchText} = Params) -> + Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText), + start_id => make_start_id(Start, Borders), + end_id => make_end_id(End, Borders), + remote_bare_jid => maybe_minify_bare_jid(Host, UserJID, WithJID), + remote_resource => jid_to_non_empty_resource(WithJID)}. + +make_start_id(Start, Borders) -> StartID = maybe_encode_compact_uuid(Start, 0), - EndID = maybe_encode_compact_uuid(End, 255), - StartID2 = apply_start_border(Borders, StartID), - EndID2 = apply_end_border(Borders, EndID), - prepare_filters(UserID, StartID2, EndID2, MinWithJID, WithResource, SearchText). - --spec prepare_filters(UserID :: non_neg_integer(), - StartID :: mod_mam:message_id() | undefined, - EndID :: mod_mam:message_id() | undefined, - WithJID :: binary() | undefined, - WithResource :: binary() | undefined, - SearchText :: binary() | undefined) -> filter(). -prepare_filters(UserID, StartID, EndID, WithJID, WithResource, SearchText) -> - [{equal, user_id, UserID}] ++ - case StartID of - undefined -> []; - _ -> [{ge, id, StartID}] - end ++ - case EndID of - undefined -> []; - _ -> [{le, id, EndID}] - end ++ - case WithJID of - undefined -> []; - _ -> [{equal, remote_bare_jid, WithJID}] - end ++ - case WithResource of - undefined -> []; - _ -> [{equal, remote_resource, WithResource}] - end ++ - case SearchText of - undefined -> []; - _ -> prepare_search_filters(SearchText) + apply_start_border(Borders, StartID). + +make_end_id(End, Borders) -> + EndID = maybe_encode_compact_uuid(End, 255), + apply_end_border(Borders, EndID). + +jid_to_non_empty_resource(#jid{lresource = Res}) when byte_size(Res) > 0 -> + Res; +jid_to_non_empty_resource(_) -> + undefined. + +-record(lookup_field, {op, column, param, required, value_maker}). + +lookup_fields() -> + [#lookup_field{op = equal, column = user_id, param = archive_id, required = true}, + #lookup_field{op = ge, column = id, param = start_id}, + #lookup_field{op = le, column = id, param = end_id}, + #lookup_field{op = equal, column = remote_bare_jid, param = remote_bare_jid}, + #lookup_field{op = equal, column = remote_resource, param = remote_resource}, + #lookup_field{op = like, column = search_body, param = norm_search_text, value_maker = search_words}]. + +prepare_filter(Params) -> + [new_filter(Field, Value) + || Field <- lookup_fields(), + Value <- field_to_values(Field, Params)]. + +field_to_values(#lookup_field{param = Param, value_maker = ValueMaker, required = Required} = Field, Params) -> + case maps:find(Param, Params) of + {ok, Value} when Value =/= undefined -> + make_value(ValueMaker, Value); + Other when Required -> + error(#{reason => missing_required_field, field => Field, params => Params}); + _ -> + [] end. +make_value(search_words, Value) -> + search_words(Value); +make_value(undefined, Value) -> + [Value]. + +new_filter(#lookup_field{op = Op, column = Column}, Value) -> + {Op, Column, Value}. + %% Constructs a separate LIKE filter for each word. %% SearchText example is "word1%word2%word3". %% Order of words does not matter (they can go in any order). -prepare_search_filters(SearchText) -> +search_words(SearchText) -> Words = binary:split(SearchText, <<"%">>, [global]), - [prepare_search_filter(Word) || Word <- Words]. - --spec prepare_search_filter(binary()) -> filter_field(). -prepare_search_filter(Word) -> - %% Search for "%Word%" - {like, search_body, <<"%", Word/binary, "%">>}. + [<<"%", Word/binary, "%">> || Word <- lists:sublist(Words, ?SEARCH_WORDS_LIMIT)]. From 75dac6a55210e80349a12a07b1cb9409ee031f45 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 14:47:27 +0100 Subject: [PATCH 09/92] Refactor opt_count logic (less black magic) --- src/mam/mod_mam_rdbms_arch.erl | 154 +++++++++++++++------------------ 1 file changed, 71 insertions(+), 83 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index a7b64ba00a..6a46b898cd 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -265,10 +265,10 @@ lookup_messages({error, _Reason}=Result, _Host, _Params) -> lookup_messages(_Result, Host, Params) -> try RSM = maps:get(rsm, Params), - Params2 = Params#{opt_count_supported => is_opt_count_supported_for(RSM)}, + Params2 = Params#{opt_count_type => opt_count_type(RSM)}, Params3 = extend_params(Host, Params2), Filter = prepare_filter(Params3), - do_lookup_messages(Host, Filter, Params3) + choose_lookup_messages_strategy(Host, Filter, Params3) catch _Type:Reason:S -> {error, {Reason, {stacktrace, S}}} end. @@ -276,33 +276,39 @@ lookup_messages(_Result, Host, Params) -> %% Not supported: %% - #rsm_in{direction = aft, id = ID} %% - #rsm_in{direction = before, id = ID} -is_opt_count_supported_for(#rsm_in{direction = before, id = undefined}) -> - true; %% last page is supported -is_opt_count_supported_for(#rsm_in{direction = undefined}) -> - true; %% offset -is_opt_count_supported_for(undefined) -> - true; %% no RSM -is_opt_count_supported_for(_) -> - false. +opt_count_type(#rsm_in{direction = before, id = undefined}) -> + last_page; %% last page is supported +opt_count_type(#rsm_in{direction = undefined}) -> + by_offset; %% offset +opt_count_type(undefined) -> + by_offset; %% no RSM +opt_count_type(_) -> + none. %% id field is defined in RSM %% There are several strategies how to extract messages: %% - we can use regular query that requires counting; %% - we can reduce number of queries if we skip counting for small data sets; %% - sometimes we want not to count at all %% (for example, our client side counts ones and keep the information) -do_lookup_messages(Host, Filter, Params = #{owner_jid := UserJID, rsm := RSM, - page_size := PageSize}) -> +choose_lookup_messages_strategy(Host, Filter, + Params = #{owner_jid := UserJID, rsm := RSM, + page_size := PageSize}) -> case Params of #{is_simple := true} -> %% Simple query without calculating offset and total count lookup_messages_simple(Host, UserJID, RSM, PageSize, Filter); - #{is_simple := opt_count, opt_count_supported := true} -> - %% Extract messages first than calculate offset and total count - %% Useful for small result sets (less than one page, than one query is enough) - lookup_messages_opt_count(Host, UserJID, RSM, PageSize, Filter); + %% NOTICE: We always prefer opt_count optimization, if possible. + %% Clients don't event know what opt_count is. + #{opt_count_type := last_page} when PageSize > 0 -> + %% Extract messages before calculating offset and total count + %% Useful for small result sets + lookup_last_page(Host, UserJID, PageSize, Filter); + #{opt_count_type := by_offset} when PageSize > 0 -> + %% Extract messages before calculating offset and total count + %% Useful for small result sets + lookup_by_offset(Host, UserJID, RSM, PageSize, Filter); _ -> - %% Unsupported opt_count or just a regular query - %% Calculate offset and total count first than extract messages + %% Calculate offset and total count first before extracting messages lookup_messages_regular(Host, UserJID, RSM, PageSize, Filter) end. @@ -324,57 +330,45 @@ rsm_to_filter(RSM, Filter) -> {Filter, 0, asc} end. -%% Cases that cannot be optimized and used with this function: -%% - #rsm_in{direction = aft, id = ID} -%% - #rsm_in{direction = before, id = ID} -lookup_messages_opt_count(Host, UserJID, - #rsm_in{direction = before, id = undefined}, - PageSize, Filter) -> - %% Last page +%% This function handles case: #rsm_in{direction = before, id = undefined} +%% Assumes assert_rsm_without_id(RSM) +lookup_last_page(Host, UserJID, PageSize, Filter) -> MessageRows = extract_messages(Host, Filter, 0, PageSize, desc), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {MessageRowsCount, 0, - rows_to_uniform_format(Host, UserJID, MessageRows)}}; - false -> - FirstID = row_to_message_id(hd(MessageRows)), - Offset = calc_count(Host, before_id(FirstID, Filter)), - {ok, {Offset + MessageRowsCount, Offset, - rows_to_uniform_format(Host, UserJID, MessageRows)}} - end; -lookup_messages_opt_count(Host, UserJID, - #rsm_in{direction = undefined, index = Offset}, - PageSize, Filter) -> - %% By offset + Messages = rows_to_uniform_format(Host, UserJID, MessageRows), + SelectedCount = length(MessageRows), + Offset = + case SelectedCount < PageSize of + true -> + 0; %% Result fits on a single page + false -> + FirstID = row_to_message_id(hd(MessageRows)), + calc_count(Host, before_id(FirstID, Filter)) + end, + {ok, {Offset + SelectedCount, Offset, Messages}}. + +lookup_by_offset(Host, UserJID, RSM, PageSize, Filter) -> + assert_rsm_without_id(RSM), + Offset = rsm_to_index(RSM), MessageRows = extract_messages(Host, Filter, Offset, PageSize, asc), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {Offset + MessageRowsCount, Offset, - rows_to_uniform_format(Host, UserJID, MessageRows)}}; - false -> - LastID = row_to_message_id(lists:last(MessageRows)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), - {ok, {Offset + MessageRowsCount + CountAfterLastID, Offset, - rows_to_uniform_format(Host, UserJID, MessageRows)}} - end; -lookup_messages_opt_count(Host, UserJID, - undefined, - PageSize, Filter) -> - %% First page - MessageRows = extract_messages(Host, Filter, 0, PageSize, asc), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {MessageRowsCount, 0, - rows_to_uniform_format(Host, UserJID, MessageRows)}}; - false -> - LastID = row_to_message_id(lists:last(MessageRows)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), - {ok, {MessageRowsCount + CountAfterLastID, 0, - rows_to_uniform_format(Host, UserJID, MessageRows)}} - end. + Messages = rows_to_uniform_format(Host, UserJID, MessageRows), + SelectedCount = length(MessageRows), + TotalCount = + case SelectedCount < PageSize of + true -> + Offset + SelectedCount; %% Result fits on a single page + false -> + LastID = row_to_message_id(lists:last(MessageRows)), + CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), + Offset + SelectedCount + CountAfterLastID + end, + {ok, {TotalCount, Offset, Messages}}. + +assert_rsm_without_id(undefined) -> ok; +assert_rsm_without_id(#rsm_in{id = undefined}) -> ok. + +rsm_to_index(#rsm_in{direction = undefined, index = Offset}) + when is_integer(Offset) -> Offset; +rsm_to_index(_) -> 0. lookup_messages_regular(Host, UserJID, RSM, PageSize, Filter) -> TotalCount = calc_count(Host, Filter), @@ -441,20 +435,20 @@ remove_archive(Host, UserID) -> %% Columns are `["id", "from_jid", "message"]'. -type msg() :: {binary(), jid:literal_jid(), binary()}. -spec extract_messages(Host :: jid:server(), - Filter :: filter(), IOffset :: non_neg_integer(), IMax :: pos_integer(), + Filter :: filter(), Offset :: non_neg_integer(), Max :: pos_integer(), Order :: asc | desc) -> [msg()]. -extract_messages(_Host, _Filter, _IOffset, 0, _) -> +extract_messages(_Host, _Filter, _Offset, 0, _) -> []; -extract_messages(Host, Filter, IOffset, IMax, Order) -> +extract_messages(Host, Filter, Offset, Max, Order) -> {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, Order), + do_extract_messages(Host, Filter, Offset, Max, Order), ?LOG_DEBUG(#{what => mam_extract_messages, - mam_filter => Filter, offset => IOffset, max => IMax, + mam_filter => Filter, offset => Offset, max => Max, host => Host, message_rows => MessageRows}), maybe_reverse(Order, MessageRows). -do_extract_messages(Host, Filters, IOffset, IMax, Order) -> - Filters2 = Filters ++ rdbms_queries:limit_offset_filters(IMax, IOffset), +do_extract_messages(Host, Filters, Offset, Max, Order) -> + Filters2 = Filters ++ rdbms_queries:limit_offset_filters(Max, Offset), do_lookup_query(lookup, Host, Filters2, Order). do_lookup_query(QueryType, Host, Filters, Order) -> @@ -515,15 +509,9 @@ calc_before(Host, Filter, ID) -> -spec calc_count(Host :: jid:server(), Filter :: filter()) -> non_neg_integer(). calc_count(Host, Filter) -> - {selected, [{BCount}]} = - do_lookup_query(count, Host, Filter, unordered), - mongoose_rdbms:result_to_integer(BCount). - -%% @doc #rsm_in{ -%% max = non_neg_integer() | undefined, -%% direction = before | aft | undefined, -%% id = binary() | undefined, -%% index = non_neg_integer() | undefined} + Result = do_lookup_query(count, Host, Filter, unordered), + mongoose_rdbms:selected_to_integer(Result). + -spec calc_offset(Host :: jid:server(), Filter :: filter(), PageSize :: non_neg_integer(), TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in()) -> non_neg_integer(). From 57787c7d229cc3d0128f081374e44c1ca7dafb7b Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 17:22:30 +0100 Subject: [PATCH 10/92] Add Index Hint --- src/mam/mod_mam.erl | 5 +- src/mam/mod_mam_rdbms_arch.erl | 196 ++++++++++++++++----------------- 2 files changed, 98 insertions(+), 103 deletions(-) diff --git a/src/mam/mod_mam.erl b/src/mam/mod_mam.erl index ddecc35fa7..d10a633163 100644 --- a/src/mam/mod_mam.erl +++ b/src/mam/mod_mam.erl @@ -118,9 +118,11 @@ -type archive_id() :: non_neg_integer(). -type borders() :: #mam_borders{}. + +-type message_row() :: {message_id(), jid:jid(), exml:element()}. -type lookup_result() :: {TotalCount :: non_neg_integer() | undefined, Offset :: non_neg_integer() | undefined, - MessageRows :: [{message_id(), jid:jid(), exml:element()}]}. + MessageRows :: [message_row()]}. %% Internal types -type iterator_fun() :: fun(() -> {'ok', {_, _}}). @@ -150,6 +152,7 @@ unix_timestamp/0, archive_id/0, lookup_result/0, + message_row/0, message_id/0, restore_option/0, archive_message_params/0 diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 6a46b898cd..9bea73cc8e 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -158,7 +158,7 @@ encode_direction(outgoing) -> <<"O">>. -spec archive_size(Size :: integer(), Host :: jid:server(), ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). -archive_size(Size, Host, UserID, _UserJID) when is_integer(Size) -> +archive_size(Size, Host, UserID, _ArcJID) when is_integer(Size) -> Result = mod_mam_utils:success_sql_execute(Host, mam_archive_size, [UserID]), mongoose_rdbms:selected_to_integer(Result). @@ -264,11 +264,9 @@ lookup_messages({error, _Reason}=Result, _Host, _Params) -> Result; lookup_messages(_Result, Host, Params) -> try - RSM = maps:get(rsm, Params), - Params2 = Params#{opt_count_type => opt_count_type(RSM)}, - Params3 = extend_params(Host, Params2), - Filter = prepare_filter(Params3), - choose_lookup_messages_strategy(Host, Filter, Params3) + ExtParams = extend_params(Host, Params), + Filter = prepare_filter(ExtParams), + choose_lookup_messages_strategy(Host, Filter, ExtParams) catch _Type:Reason:S -> {error, {Reason, {stacktrace, S}}} end. @@ -291,31 +289,31 @@ opt_count_type(_) -> %% - sometimes we want not to count at all %% (for example, our client side counts ones and keep the information) choose_lookup_messages_strategy(Host, Filter, - Params = #{owner_jid := UserJID, rsm := RSM, + Params = #{owner_jid := ArcJID, rsm := RSM, page_size := PageSize}) -> case Params of #{is_simple := true} -> %% Simple query without calculating offset and total count - lookup_messages_simple(Host, UserJID, RSM, PageSize, Filter); + lookup_messages_simple(Host, ArcJID, RSM, PageSize, Filter); %% NOTICE: We always prefer opt_count optimization, if possible. %% Clients don't event know what opt_count is. #{opt_count_type := last_page} when PageSize > 0 -> %% Extract messages before calculating offset and total count %% Useful for small result sets - lookup_last_page(Host, UserJID, PageSize, Filter); + lookup_last_page(Host, ArcJID, PageSize, Filter); #{opt_count_type := by_offset} when PageSize > 0 -> %% Extract messages before calculating offset and total count %% Useful for small result sets - lookup_by_offset(Host, UserJID, RSM, PageSize, Filter); + lookup_by_offset(Host, ArcJID, RSM, PageSize, Filter); _ -> %% Calculate offset and total count first before extracting messages - lookup_messages_regular(Host, UserJID, RSM, PageSize, Filter) + lookup_messages_regular(Host, ArcJID, RSM, PageSize, Filter) end. -lookup_messages_simple(Host, UserJID, RSM, PageSize, Filter) -> +lookup_messages_simple(Host, ArcJID, RSM, PageSize, Filter) -> {Filter2, Offset, Order} = rsm_to_filter(RSM, Filter), - MessageRows = extract_messages(Host, Filter2, Offset, PageSize, Order), - {ok, {undefined, undefined, rows_to_uniform_format(Host, UserJID, MessageRows)}}. + Messages = extract_messages(Host, ArcJID, Filter2, Offset, PageSize, Order), + {ok, {undefined, undefined, Messages}}. rsm_to_filter(RSM, Filter) -> case RSM of @@ -332,34 +330,32 @@ rsm_to_filter(RSM, Filter) -> %% This function handles case: #rsm_in{direction = before, id = undefined} %% Assumes assert_rsm_without_id(RSM) -lookup_last_page(Host, UserJID, PageSize, Filter) -> - MessageRows = extract_messages(Host, Filter, 0, PageSize, desc), - Messages = rows_to_uniform_format(Host, UserJID, MessageRows), - SelectedCount = length(MessageRows), +lookup_last_page(Host, ArcJID, PageSize, Filter) -> + Messages = extract_messages(Host, ArcJID, Filter, 0, PageSize, desc), + Selected = length(Messages), Offset = - case SelectedCount < PageSize of + case Selected < PageSize of true -> 0; %% Result fits on a single page false -> - FirstID = row_to_message_id(hd(MessageRows)), + FirstID = uniform_to_message_id(hd(Messages)), calc_count(Host, before_id(FirstID, Filter)) end, - {ok, {Offset + SelectedCount, Offset, Messages}}. + {ok, {Offset + Selected, Offset, Messages}}. -lookup_by_offset(Host, UserJID, RSM, PageSize, Filter) -> +lookup_by_offset(Host, ArcJID, RSM, PageSize, Filter) -> assert_rsm_without_id(RSM), Offset = rsm_to_index(RSM), - MessageRows = extract_messages(Host, Filter, Offset, PageSize, asc), - Messages = rows_to_uniform_format(Host, UserJID, MessageRows), - SelectedCount = length(MessageRows), + Messages = extract_messages(Host, ArcJID, Filter, Offset, PageSize, asc), + Selected = length(Messages), TotalCount = - case SelectedCount < PageSize of + case Selected < PageSize of true -> - Offset + SelectedCount; %% Result fits on a single page + Offset + Selected; %% Result fits on a single page false -> - LastID = row_to_message_id(lists:last(MessageRows)), + LastID = uniform_to_message_id(lists:last(Messages)), CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), - Offset + SelectedCount + CountAfterLastID + Offset + Selected + CountAfterLastID end, {ok, {TotalCount, Offset, Messages}}. @@ -370,19 +366,19 @@ rsm_to_index(#rsm_in{direction = undefined, index = Offset}) when is_integer(Offset) -> Offset; rsm_to_index(_) -> 0. -lookup_messages_regular(Host, UserJID, RSM, PageSize, Filter) -> +lookup_messages_regular(Host, ArcJID, RSM, PageSize, Filter) -> TotalCount = calc_count(Host, Filter), Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = + Messages = case RSM of #rsm_in{direction = aft, id = ID} -> - extract_messages(Host, from_id(ID, Filter), 0, PageSize + 1, asc); + extract_messages(Host, ArcJID, from_id(ID, Filter), 0, PageSize + 1, asc); #rsm_in{direction = before, id = ID} when ID =/= undefined -> - extract_messages(Host, to_id(ID, Filter), 0, PageSize + 1, desc); + extract_messages(Host, ArcJID, to_id(ID, Filter), 0, PageSize + 1, desc); _ -> - extract_messages(Host, Filter, Offset, PageSize, asc) + extract_messages(Host, ArcJID, Filter, Offset, PageSize, asc) end, - Result = {TotalCount, Offset, rows_to_uniform_format(Host, UserJID, MessageRows)}, + Result = {TotalCount, Offset, Messages}, mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result). -spec after_id(ID :: mod_mam:message_id(), Filter :: filter()) -> filter(). @@ -404,18 +400,17 @@ from_id(ID, Filter) -> to_id(ID, Filter) -> [{le, id, ID}|Filter]. -rows_to_uniform_format(Host, UserJID, MessageRows) -> - [do_row_to_uniform_format(Host, UserJID, Row) || Row <- MessageRows]. +rows_to_uniform_format(Host, ArcJID, MessageRows) -> + [row_to_uniform_format(Host, ArcJID, Row) || Row <- MessageRows]. -do_row_to_uniform_format(Host, UserJID, {BMessID, BSrcJID, SDataRaw}) -> +row_to_uniform_format(Host, ArcJID, {BMessID, BSrcJID, SDataRaw}) -> MessID = mongoose_rdbms:result_to_integer(BMessID), - SrcJID = stored_binary_to_jid(Host, UserJID, BSrcJID), + SrcJID = stored_binary_to_jid(Host, ArcJID, BSrcJID), Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), Packet = stored_binary_to_packet(Host, Data), {MessID, SrcJID, Packet}. -row_to_message_id({BMessID, _, _}) -> - mongoose_rdbms:result_to_integer(BMessID). +uniform_to_message_id({MessID, _, _}) -> MessID. %% Removals @@ -423,7 +418,7 @@ row_to_message_id({BMessID, _, _}) -> -spec remove_archive(Acc :: mongoose_acc:t(), Host :: jid:server(), ArchiveID :: mod_mam:archive_id(), RoomJID :: jid:jid()) -> mongoose_acc:t(). -remove_archive(Acc, Host, UserID, _UserJID) -> +remove_archive(Acc, Host, UserID, _ArcJID) -> remove_archive(Host, UserID), Acc. @@ -433,30 +428,29 @@ remove_archive(Host, UserID) -> %% @doc Each record is a tuple of form %% `{<<"13663125233">>, <<"bob@localhost">>, <>}'. %% Columns are `["id", "from_jid", "message"]'. --type msg() :: {binary(), jid:literal_jid(), binary()}. --spec extract_messages(Host :: jid:server(), +-spec extract_messages(Host :: jid:server(), ArcJID :: jid:jid(), Filter :: filter(), Offset :: non_neg_integer(), Max :: pos_integer(), - Order :: asc | desc) -> [msg()]. -extract_messages(_Host, _Filter, _Offset, 0, _) -> + Order :: asc | desc) -> [mod_mam:message_row()]. +extract_messages(_Host, _ArcJID, _Filter, _Offset, 0, _) -> []; -extract_messages(Host, Filter, Offset, Max, Order) -> - {selected, MessageRows} = - do_extract_messages(Host, Filter, Offset, Max, Order), +extract_messages(Host, ArcJID, Filter, Offset, Max, Order) -> + {selected, MessageRows} = extract_rows(Host, Filter, Offset, Max, Order), ?LOG_DEBUG(#{what => mam_extract_messages, mam_filter => Filter, offset => Offset, max => Max, host => Host, message_rows => MessageRows}), - maybe_reverse(Order, MessageRows). + Rows = maybe_reverse(Order, MessageRows), + rows_to_uniform_format(Host, ArcJID, Rows). -do_extract_messages(Host, Filters, Offset, Max, Order) -> +extract_rows(Host, Filters, Offset, Max, Order) -> Filters2 = Filters ++ rdbms_queries:limit_offset_filters(Max, Offset), - do_lookup_query(lookup, Host, Filters2, Order). + lookup_query(lookup, Host, Filters2, Order). -do_lookup_query(QueryType, Host, Filters, Order) -> +lookup_query(QueryType, Host, Filters, Order) -> StmtName = filters_to_statement_name(QueryType, Filters, Order), case mongoose_rdbms:prepared(StmtName) of false -> %% Create a new type of a query - SQL = lookup_sql_binary(QueryType, Filters, Order), + SQL = lookup_sql_binary(QueryType, Filters, Order, index_hint_sql(Host)), Columns = filters_to_columns(Filters), mongoose_rdbms:prepare(StmtName, mam_message, Columns, SQL); true -> @@ -468,13 +462,13 @@ do_lookup_query(QueryType, Host, Filters, Order) -> {selected, Rs}; Error -> What = #{what => mam_lookup_failed, statement => StmtName, - sql_query => lookup_sql_binary(QueryType, Filters, Order), + sql_query => lookup_sql_binary(QueryType, Filters, Order, index_hint_sql(Host)), reason => Error, host => Host}, ?LOG_ERROR(What), error(What) catch Class:Error:Stacktrace -> What = #{what => mam_lookup_failed, statement => StmtName, - sql_query => lookup_sql_binary(QueryType, Filters, Order), + sql_query => lookup_sql_binary(QueryType, Filters, Order, index_hint_sql(Host)), class => Class, stacktrace => Stacktrace, reason => Error, host => Host}, ?LOG_ERROR(What), @@ -509,26 +503,26 @@ calc_before(Host, Filter, ID) -> -spec calc_count(Host :: jid:server(), Filter :: filter()) -> non_neg_integer(). calc_count(Host, Filter) -> - Result = do_lookup_query(count, Host, Filter, unordered), + Result = lookup_query(count, Host, Filter, unordered), mongoose_rdbms:selected_to_integer(Result). -spec calc_offset(Host :: jid:server(), Filter :: filter(), PageSize :: non_neg_integer(), TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in()) -> non_neg_integer(). -calc_offset(_LS, _F, _PS, _TC, #rsm_in{direction = undefined, index = Index}) - when is_integer(Index) -> - Index; -%% Requesting the Last Page in a Result Set -calc_offset(_LS, _F, PS, TC, #rsm_in{direction = before, id = undefined}) -> - max(0, TC - PS); -calc_offset(Host, F, PS, _TC, #rsm_in{direction = before, id = ID}) - when is_integer(ID) -> - max(0, calc_before(Host, F, ID) - PS); -calc_offset(Host, F, _PS, _TC, #rsm_in{direction = aft, id = ID}) - when is_integer(ID) -> - calc_index(Host, F, ID); -calc_offset(_LS, _F, _PS, _TC, _RSM) -> - 0. +calc_offset(Host, Filter, PageSize, TotalCount, RSM) -> + case RSM of + #rsm_in{direction = undefined, index = Index} when is_integer(Index) -> + Index; + #rsm_in{direction = before, id = undefined} -> + %% Requesting the Last Page in a Result Set + max(0, TotalCount - PageSize); + #rsm_in{direction = before, id = ID} when is_integer(ID) -> + max(0, calc_before(Host, Filter, ID) - PageSize); + #rsm_in{direction = aft, id = ID} when is_integer(ID) -> + calc_index(Host, Filter, ID); + _ -> + 0 + end. maybe_reverse(asc, List) -> List; @@ -538,6 +532,9 @@ maybe_reverse(desc, List) -> extract_gdpr_messages(Host, ArchiveID) -> mod_mam_utils:success_sql_execute(Host, mam_extract_gdpr_messages, [ArchiveID]). +%% ---------------------------------------------------------------------- +%% Optimizations + maybe_minify_bare_jid(_Host, _LocJID, undefined) -> undefined; maybe_minify_bare_jid(Host, LocJID, JID) -> @@ -546,21 +543,26 @@ maybe_minify_bare_jid(Host, LocJID, JID) -> minify_bare_jid(Host, LocJID, JID) -> jid_to_stored_binary(Host, LocJID, jid:to_bare(JID)). +make_start_id(Start, Borders) -> + StartID = maybe_encode_compact_uuid(Start, 0), + apply_start_border(Borders, StartID). + +make_end_id(End, Borders) -> + EndID = maybe_encode_compact_uuid(End, 255), + apply_end_border(Borders, EndID). + maybe_encode_compact_uuid(undefined, _) -> undefined; maybe_encode_compact_uuid(Microseconds, NodeID) -> encode_compact_uuid(Microseconds, NodeID). -%% ---------------------------------------------------------------------- -%% Optimizations - -jid_to_stored_binary(Host, UserJID, JID) -> +jid_to_stored_binary(Host, ArcJID, JID) -> Module = db_jid_codec(Host), - mam_jid:encode(Module, UserJID, JID). + mam_jid:encode(Module, ArcJID, JID). -stored_binary_to_jid(Host, UserJID, BSrcJID) -> +stored_binary_to_jid(Host, ArcJID, BSrcJID) -> Module = db_jid_codec(Host), - mam_jid:decode(Module, UserJID, BSrcJID). + mam_jid:decode(Module, ArcJID, BSrcJID). packet_to_stored_binary(Host, Packet) -> Module = db_message_codec(Host), @@ -578,32 +580,29 @@ db_jid_codec(Host) -> db_message_codec(Host) -> gen_mod:get_module_opt(Host, ?MODULE, db_message_format, mam_message_compressed_eterm). -%gdpr helpers -gdpr_decode_jid(Host, UserJID, BSrcJID) -> - Codec = mod_mam_meta:get_mam_module_opt(Host, ?MODULE, db_jid_format, mam_jid_mini), - JID = mam_jid:decode(Codec, UserJID, BSrcJID), +%% GDPR helpers +gdpr_decode_jid(Host, ArcJID, BSrcJID) -> + JID = stored_binary_to_jid(Host, ArcJID, BSrcJID), jid:to_binary(JID). gdpr_decode_packet(Host, SDataRaw) -> - Codec = mod_mam_meta:get_mam_module_opt(Host, ?MODULE, db_message_format, - mam_message_compressed_eterm), - Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Message = mam_message:decode(Codec, Data), + Bin = mongoose_rdbms:unescape_binary(Host, SDataRaw), + Message = stored_binary_to_packet(Host, Bin), exml:to_binary(Message). %% ---------------------------------------------------------------------- %% Prepared queries helpers -lookup_sql_binary(QueryType, Filters, Order) -> - iolist_to_binary(lookup_sql(QueryType, Filters, Order)). +lookup_sql_binary(QueryType, Filters, Order, IndexHintSQL) -> + iolist_to_binary(lookup_sql(QueryType, Filters, Order, IndexHintSQL)). -lookup_sql(QueryType, Filters, Order) -> +lookup_sql(QueryType, Filters, Order, IndexHintSQL) -> LimitSQL = limit_sql(QueryType), OrderSQL = order_to_sql(Order), FilterSQL = filters_to_sql(Filters), ["SELECT ", columns_sql(QueryType), " " "FROM mam_message ", - FilterSQL, OrderSQL, LimitSQL]. + IndexHintSQL, FilterSQL, OrderSQL, LimitSQL]. columns_sql(lookup) -> "id, from_jid, message"; columns_sql(count) -> "COUNT(*)". @@ -684,23 +683,16 @@ filter_to_sql(equal, Column) -> filter_to_sql(like, Column) -> Column ++ " LIKE ?". -extend_params(Host, #{owner_jid := UserJID, borders := Borders, +extend_params(Host, #{owner_jid := ArcJID, rsm := RSM, borders := Borders, start_ts := Start, end_ts := End, with_jid := WithJID, search_text := SearchText} = Params) -> - Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText), + Params#{opt_count_type => opt_count_type(RSM), + norm_search_text => mod_mam_utils:normalize_search_text(SearchText), start_id => make_start_id(Start, Borders), end_id => make_end_id(End, Borders), - remote_bare_jid => maybe_minify_bare_jid(Host, UserJID, WithJID), + remote_bare_jid => maybe_minify_bare_jid(Host, ArcJID, WithJID), remote_resource => jid_to_non_empty_resource(WithJID)}. -make_start_id(Start, Borders) -> - StartID = maybe_encode_compact_uuid(Start, 0), - apply_start_border(Borders, StartID). - -make_end_id(End, Borders) -> - EndID = maybe_encode_compact_uuid(End, 255), - apply_end_border(Borders, EndID). - jid_to_non_empty_resource(#jid{lresource = Res}) when byte_size(Res) > 0 -> Res; jid_to_non_empty_resource(_) -> @@ -726,7 +718,7 @@ field_to_values(#lookup_field{param = Param, value_maker = ValueMaker, required {ok, Value} when Value =/= undefined -> make_value(ValueMaker, Value); Other when Required -> - error(#{reason => missing_required_field, field => Field, params => Params}); + error(#{reason => missing_required_field, field => Field, params => Params, result => Other}); _ -> [] end. From 3b3148ef5f4b4e5e5e4d27f57b4585c50847bcbd Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 17:28:14 +0100 Subject: [PATCH 11/92] Reorder code --- src/mam/mod_mam_rdbms_arch.erl | 36 +++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 9bea73cc8e..81cd4e8186 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -189,6 +189,7 @@ do_archive_message(Host, Params) -> {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_message, Row), retract_message(Host, Params). +%% Retraction logic -spec retract_message(jid:server(), mod_mam:archive_message_params()) -> ok. retract_message(Host, #{archive_id := UserID, local_jid := LocJID, @@ -229,6 +230,7 @@ execute_make_tombstone(Host, TombstoneData, UserID, MessID) -> mod_mam_utils:success_sql_execute(Host, mam_make_tombstone, [TombstoneData, UserID, MessID]). +%% Insert logic -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). prepare_message(Host, #{message_id := MessID, archive_id := UserID, @@ -258,6 +260,24 @@ prepare_insert(Name, NumRows) -> mongoose_rdbms:prepare(Name, Table, Fields2, Query), ok. + +%% Removal logic +-spec remove_archive(Acc :: mongoose_acc:t(), Host :: jid:server(), + ArchiveID :: mod_mam:archive_id(), + RoomJID :: jid:jid()) -> mongoose_acc:t(). +remove_archive(Acc, Host, UserID, _ArcJID) -> + remove_archive(Host, UserID), + Acc. + +remove_archive(Host, UserID) -> + {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_archive_remove, [UserID]). + +%% GDPR logic +extract_gdpr_messages(Host, ArchiveID) -> + mod_mam_utils:success_sql_execute(Host, mam_extract_gdpr_messages, [ArchiveID]). + + +%% Lookup logic -spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> {ok, mod_mam:lookup_result()}. lookup_messages({error, _Reason}=Result, _Host, _Params) -> @@ -412,19 +432,6 @@ row_to_uniform_format(Host, ArcJID, {BMessID, BSrcJID, SDataRaw}) -> uniform_to_message_id({MessID, _, _}) -> MessID. - -%% Removals - --spec remove_archive(Acc :: mongoose_acc:t(), Host :: jid:server(), - ArchiveID :: mod_mam:archive_id(), - RoomJID :: jid:jid()) -> mongoose_acc:t(). -remove_archive(Acc, Host, UserID, _ArcJID) -> - remove_archive(Host, UserID), - Acc. - -remove_archive(Host, UserID) -> - {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_archive_remove, [UserID]). - %% @doc Each record is a tuple of form %% `{<<"13663125233">>, <<"bob@localhost">>, <>}'. %% Columns are `["id", "from_jid", "message"]'. @@ -529,9 +536,6 @@ maybe_reverse(asc, List) -> maybe_reverse(desc, List) -> lists:reverse(List). -extract_gdpr_messages(Host, ArchiveID) -> - mod_mam_utils:success_sql_execute(Host, mam_extract_gdpr_messages, [ArchiveID]). - %% ---------------------------------------------------------------------- %% Optimizations From 332f54d3ed5e34fdb40b7d192362063e6a120cdc Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 18:12:36 +0100 Subject: [PATCH 12/92] Add db_mappings --- src/mam/mod_mam_rdbms_arch.erl | 147 +++++++++++++++++++-------------- 1 file changed, 86 insertions(+), 61 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 81cd4e8186..ab6d221606 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -112,9 +112,10 @@ get_mam_pm_gdpr_data(Acc, #jid{ user = User, server = Server, lserver = LServer case mod_mam:archive_id(Server, User) of undefined -> []; ArchiveID -> + Module = ?MODULE, {selected, Rows} = extract_gdpr_messages(LServer, ArchiveID), - [{BMessID, gdpr_decode_jid(LServer, UserJid, FromJID), - gdpr_decode_packet(LServer, SDataRaw)} || {BMessID, FromJID, SDataRaw} <- Rows] ++ Acc + [{BMessID, gdpr_decode_jid(LServer, Module, UserJid, FromJID), + gdpr_decode_packet(LServer, Module, SDataRaw)} || {BMessID, FromJID, SDataRaw} <- Rows] ++ Acc end. %% ---------------------------------------------------------------------- @@ -202,7 +203,7 @@ retract_message(Host, #{archive_id := UserID, end. retract_message(Host, UserID, LocJID, RemJID, OriginID, Dir) -> - MinBareRemJID = minify_bare_jid(Host, LocJID, RemJID), + MinBareRemJID = minify_bare_jid(Host, ?MODULE, LocJID, RemJID), BinDir = encode_direction(Dir), {selected, Rows} = execute_select_messages_to_retract( Host, UserID, MinBareRemJID, OriginID, BinDir), @@ -215,10 +216,10 @@ make_tombstone(_Host, UserID, OriginID, []) -> user_id => UserID, origin_id => OriginID}); make_tombstone(Host, UserID, OriginID, [{ResMessID, ResData}]) -> Data = mongoose_rdbms:unescape_binary(Host, ResData), - Packet = stored_binary_to_packet(Host, Data), + Packet = stored_binary_to_packet(Host, ?MODULE, Data), MessID = mongoose_rdbms:result_to_integer(ResMessID), Tombstone = mod_mam_utils:tombstone(Packet, OriginID), - TombstoneData = packet_to_stored_binary(Host, Tombstone), + TombstoneData = packet_to_stored_binary(Host, ?MODULE, Tombstone), execute_make_tombstone(Host, TombstoneData, UserID, MessID). execute_select_messages_to_retract(Host, UserID, BareRemJID, OriginID, Dir) -> @@ -231,31 +232,54 @@ execute_make_tombstone(Host, TombstoneData, UserID, MessID) -> [TombstoneData, UserID, MessID]). %% Insert logic + +-record(db_mapping, {column, param, format, module}). + +db_mappings() -> + [#db_mapping{column = id, param = message_id, format = int}, + #db_mapping{column = user_id, param = archive_id, format = int}, + #db_mapping{column = remote_bare_jid, param = remote_jid, format = bare_jid, module = ?MODULE}, + #db_mapping{column = remote_resource, param = remote_jid, format = jid_resource}, + #db_mapping{column = direction, param = direction, format = direction}, + #db_mapping{column = from_jid, param = source_jid, format = jid, module = ?MODULE}, + #db_mapping{column = origin_id, param = origin_id, format = maybe_binary}, + #db_mapping{column = message, param = packet, format = xml, module = ?MODULE}, + #db_mapping{column = search_body, param = packet, format = search, module = mod_mam}]. + -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). -prepare_message(Host, #{message_id := MessID, - archive_id := UserID, - local_jid := LocJID, - remote_jid := RemJID = #jid{lresource = RemLResource}, - source_jid := SrcJID, - origin_id := OriginID, - direction := Dir, - packet := Packet}) -> - SBareRemJID = jid_to_stored_binary(Host, LocJID, jid:to_bare(RemJID)), - SSrcJID = jid_to_stored_binary(Host, LocJID, SrcJID), - SDir = encode_direction(Dir), - SOriginID = case OriginID of - none -> null; - _ -> OriginID - end, - Data = packet_to_stored_binary(Host, Packet), - TextBody = mod_mam_utils:packet_to_search_body(mod_mam, Host, Packet), - [MessID, UserID, SBareRemJID, RemLResource, SDir, SSrcJID, SOriginID, Data, TextBody]. +prepare_message(Host, Params) -> + [prepare_value(Host, Params, Mapping) || Mapping <- db_mappings()]. + +prepare_value(Host, Params, Mapping = #db_mapping{param = Param, format = Format}) -> + Value = maps:get(Param, Params), + encode_value(Format, Value, Host, Params, Mapping). + +encode_value(int, Value, _Host, _Params, _Mapping) when is_integer(Value) -> + Value; +encode_value(maybe_binary, none, _Host, _Params, _Mapping) -> + null; +encode_value(maybe_binary, Value, _Host, _Params, _Mapping) when is_binary(Value) -> + Value; +encode_value(direction, Value, _Host, _Params, _Mapping) -> + encode_direction(Value); +encode_value(bare_jid, Value, Host, #{local_jid := LocJID}, #db_mapping{module = Module}) -> + jid_to_stored_binary(Host, Module, LocJID, jid:to_bare(Value)); +encode_value(jid, Value, Host, #{local_jid := LocJID}, #db_mapping{module = Module}) -> + jid_to_stored_binary(Host, Module, LocJID, Value); +encode_value(jid_resource, #jid{lresource = Res}, Host, _Params, _Mapping) -> + Res; +encode_value(xml, Value, Host, _Params, #db_mapping{module = Module}) -> + packet_to_stored_binary(Host, Module, Value); +encode_value(search, Value, Host, _Params, #db_mapping{module = Module}) -> + mod_mam_utils:packet_to_search_body(Module, Host, Value). + +column_names() -> + [Column || #db_mapping{column = Column} <- db_mappings()]. -spec prepare_insert(Name :: atom(), NumRows :: pos_integer()) -> ok. prepare_insert(Name, NumRows) -> Table = mam_message, - Fields = [id, user_id, remote_bare_jid, remote_resource, - direction, from_jid, origin_id, message, search_body], + Fields = column_names(), {Query, Fields2} = rdbms_queries:create_bulk_insert_query(Table, Fields, NumRows), mongoose_rdbms:prepare(Name, Table, Fields2, Query), ok. @@ -314,7 +338,7 @@ choose_lookup_messages_strategy(Host, Filter, case Params of #{is_simple := true} -> %% Simple query without calculating offset and total count - lookup_messages_simple(Host, ArcJID, RSM, PageSize, Filter); + simple_lookup_messages(Host, ArcJID, RSM, PageSize, Filter); %% NOTICE: We always prefer opt_count optimization, if possible. %% Clients don't event know what opt_count is. #{opt_count_type := last_page} when PageSize > 0 -> @@ -330,7 +354,8 @@ choose_lookup_messages_strategy(Host, Filter, lookup_messages_regular(Host, ArcJID, RSM, PageSize, Filter) end. -lookup_messages_simple(Host, ArcJID, RSM, PageSize, Filter) -> +%% Just extract messages +simple_lookup_messages(Host, ArcJID, RSM, PageSize, Filter) -> {Filter2, Offset, Order} = rsm_to_filter(RSM, Filter), Messages = extract_messages(Host, ArcJID, Filter2, Offset, PageSize, Order), {ok, {undefined, undefined, Messages}}. @@ -420,14 +445,14 @@ from_id(ID, Filter) -> to_id(ID, Filter) -> [{le, id, ID}|Filter]. -rows_to_uniform_format(Host, ArcJID, MessageRows) -> - [row_to_uniform_format(Host, ArcJID, Row) || Row <- MessageRows]. +rows_to_uniform_format(Host, Module, ArcJID, MessageRows) -> + [row_to_uniform_format(Host, Module, ArcJID, Row) || Row <- MessageRows]. -row_to_uniform_format(Host, ArcJID, {BMessID, BSrcJID, SDataRaw}) -> +row_to_uniform_format(Host, Module, ArcJID, {BMessID, BSrcJID, SDataRaw}) -> MessID = mongoose_rdbms:result_to_integer(BMessID), - SrcJID = stored_binary_to_jid(Host, ArcJID, BSrcJID), + SrcJID = stored_binary_to_jid(Host, Module, ArcJID, BSrcJID), Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Packet = stored_binary_to_packet(Host, Data), + Packet = stored_binary_to_packet(Host, Module, Data), {MessID, SrcJID, Packet}. uniform_to_message_id({MessID, _, _}) -> MessID. @@ -446,7 +471,7 @@ extract_messages(Host, ArcJID, Filter, Offset, Max, Order) -> mam_filter => Filter, offset => Offset, max => Max, host => Host, message_rows => MessageRows}), Rows = maybe_reverse(Order, MessageRows), - rows_to_uniform_format(Host, ArcJID, Rows). + rows_to_uniform_format(Host, ?MODULE, ArcJID, Rows). extract_rows(Host, Filters, Offset, Max, Order) -> Filters2 = Filters ++ rdbms_queries:limit_offset_filters(Max, Offset), @@ -539,13 +564,13 @@ maybe_reverse(desc, List) -> %% ---------------------------------------------------------------------- %% Optimizations -maybe_minify_bare_jid(_Host, _LocJID, undefined) -> +maybe_minify_bare_jid(_Host, _Module, _LocJID, undefined) -> undefined; -maybe_minify_bare_jid(Host, LocJID, JID) -> - minify_bare_jid(Host, LocJID, JID). +maybe_minify_bare_jid(Host, Module, LocJID, JID) -> + minify_bare_jid(Host, Module, LocJID, JID). -minify_bare_jid(Host, LocJID, JID) -> - jid_to_stored_binary(Host, LocJID, jid:to_bare(JID)). +minify_bare_jid(Host, Module, LocJID, JID) -> + jid_to_stored_binary(Host, Module, LocJID, jid:to_bare(JID)). make_start_id(Start, Borders) -> StartID = maybe_encode_compact_uuid(Start, 0), @@ -560,38 +585,38 @@ maybe_encode_compact_uuid(undefined, _) -> maybe_encode_compact_uuid(Microseconds, NodeID) -> encode_compact_uuid(Microseconds, NodeID). -jid_to_stored_binary(Host, ArcJID, JID) -> - Module = db_jid_codec(Host), - mam_jid:encode(Module, ArcJID, JID). +jid_to_stored_binary(Host, Module, ArcJID, JID) -> + Codec = db_jid_codec(Host, Module), + mam_jid:encode(Codec, ArcJID, JID). -stored_binary_to_jid(Host, ArcJID, BSrcJID) -> - Module = db_jid_codec(Host), - mam_jid:decode(Module, ArcJID, BSrcJID). +stored_binary_to_jid(Host, Module, ArcJID, BSrcJID) -> + Codec = db_jid_codec(Host, Module), + mam_jid:decode(Codec, ArcJID, BSrcJID). -packet_to_stored_binary(Host, Packet) -> - Module = db_message_codec(Host), - mam_message:encode(Module, Packet). +packet_to_stored_binary(Host, Module, Packet) -> + Codec = db_message_codec(Host, Module), + mam_message:encode(Codec, Packet). -stored_binary_to_packet(Host, Bin) -> - Module = db_message_codec(Host), - mam_message:decode(Module, Bin). +stored_binary_to_packet(Host, Module, Bin) -> + Codec = db_message_codec(Host, Module), + mam_message:decode(Codec, Bin). --spec db_jid_codec(jid:server()) -> module(). -db_jid_codec(Host) -> - gen_mod:get_module_opt(Host, ?MODULE, db_jid_format, mam_jid_mini). +-spec db_jid_codec(jid:server(), module()) -> module(). +db_jid_codec(Host, Module) -> + gen_mod:get_module_opt(Host, Module, db_jid_format, mam_jid_mini). --spec db_message_codec(jid:server()) -> module(). -db_message_codec(Host) -> - gen_mod:get_module_opt(Host, ?MODULE, db_message_format, mam_message_compressed_eterm). +-spec db_message_codec(jid:server(), module()) -> module(). +db_message_codec(Host, Module) -> + gen_mod:get_module_opt(Host, Module, db_message_format, mam_message_compressed_eterm). %% GDPR helpers -gdpr_decode_jid(Host, ArcJID, BSrcJID) -> - JID = stored_binary_to_jid(Host, ArcJID, BSrcJID), +gdpr_decode_jid(Host, Module, ArcJID, BSrcJID) -> + JID = stored_binary_to_jid(Host, Module, ArcJID, BSrcJID), jid:to_binary(JID). -gdpr_decode_packet(Host, SDataRaw) -> +gdpr_decode_packet(Host, Module, SDataRaw) -> Bin = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Message = stored_binary_to_packet(Host, Bin), + Message = stored_binary_to_packet(Host, Module, Bin), exml:to_binary(Message). %% ---------------------------------------------------------------------- @@ -694,7 +719,7 @@ extend_params(Host, #{owner_jid := ArcJID, rsm := RSM, borders := Borders, norm_search_text => mod_mam_utils:normalize_search_text(SearchText), start_id => make_start_id(Start, Borders), end_id => make_end_id(End, Borders), - remote_bare_jid => maybe_minify_bare_jid(Host, ArcJID, WithJID), + remote_bare_jid => maybe_minify_bare_jid(Host, ?MODULE, ArcJID, WithJID), remote_resource => jid_to_non_empty_resource(WithJID)}. jid_to_non_empty_resource(#jid{lresource = Res}) when byte_size(Res) > 0 -> From 6c57bdfa5517a86443e7aa578824b761435906ab Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 18:53:33 +0100 Subject: [PATCH 13/92] Remove GDPR helpers - use standard functions --- src/mam/mod_mam_rdbms_arch.erl | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index ab6d221606..6ecd563f5c 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -108,16 +108,18 @@ stop(Host) -> -spec get_mam_pm_gdpr_data(ejabberd_gen_mam_archive:mam_pm_gdpr_data(), jid:jid()) -> ejabberd_gen_mam_archive:mam_pm_gdpr_data(). -get_mam_pm_gdpr_data(Acc, #jid{ user = User, server = Server, lserver = LServer } = UserJid) -> - case mod_mam:archive_id(Server, User) of +get_mam_pm_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> + case mod_mam:archive_id(Host, User) of undefined -> []; ArchiveID -> - Module = ?MODULE, - {selected, Rows} = extract_gdpr_messages(LServer, ArchiveID), - [{BMessID, gdpr_decode_jid(LServer, Module, UserJid, FromJID), - gdpr_decode_packet(LServer, Module, SDataRaw)} || {BMessID, FromJID, SDataRaw} <- Rows] ++ Acc + {selected, Rows} = extract_gdpr_messages(Host, ArchiveID), + Messages = rows_to_uniform_format(Host, ?MODULE, ArcJID, Rows), + [uniform_to_gdpr(M) || M <- Messages] ++ Acc end. +uniform_to_gdpr({MessID, RemoteJID, Packet}) -> + {integer_to_binary(MessID), jid:to_binary(RemoteJID), exml:to_binary(Packet)}. + %% ---------------------------------------------------------------------- %% Add hooks for mod_mam @@ -266,7 +268,7 @@ encode_value(bare_jid, Value, Host, #{local_jid := LocJID}, #db_mapping{module = jid_to_stored_binary(Host, Module, LocJID, jid:to_bare(Value)); encode_value(jid, Value, Host, #{local_jid := LocJID}, #db_mapping{module = Module}) -> jid_to_stored_binary(Host, Module, LocJID, Value); -encode_value(jid_resource, #jid{lresource = Res}, Host, _Params, _Mapping) -> +encode_value(jid_resource, #jid{lresource = Res}, _Host, _Params, _Mapping) -> Res; encode_value(xml, Value, Host, _Params, #db_mapping{module = Module}) -> packet_to_stored_binary(Host, Module, Value); @@ -609,16 +611,6 @@ db_jid_codec(Host, Module) -> db_message_codec(Host, Module) -> gen_mod:get_module_opt(Host, Module, db_message_format, mam_message_compressed_eterm). -%% GDPR helpers -gdpr_decode_jid(Host, Module, ArcJID, BSrcJID) -> - JID = stored_binary_to_jid(Host, Module, ArcJID, BSrcJID), - jid:to_binary(JID). - -gdpr_decode_packet(Host, Module, SDataRaw) -> - Bin = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Message = stored_binary_to_packet(Host, Module, Bin), - exml:to_binary(Message). - %% ---------------------------------------------------------------------- %% Prepared queries helpers From 76649b43711669d57e402f077199a79882250518 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 20:00:28 +0100 Subject: [PATCH 14/92] Make prepare_value pure by introducing env_vars --- src/mam/mod_mam_rdbms_arch.erl | 55 ++++++++++++++++++++-------------- src/mam/mod_mam_utils.erl | 18 ++++++----- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 6ecd563f5c..50191b8e40 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -234,46 +234,46 @@ execute_make_tombstone(Host, TombstoneData, UserID, MessID) -> [TombstoneData, UserID, MessID]). %% Insert logic - --record(db_mapping, {column, param, format, module}). +-record(db_mapping, {column, param, format}). db_mappings() -> [#db_mapping{column = id, param = message_id, format = int}, #db_mapping{column = user_id, param = archive_id, format = int}, - #db_mapping{column = remote_bare_jid, param = remote_jid, format = bare_jid, module = ?MODULE}, + #db_mapping{column = remote_bare_jid, param = remote_jid, format = bare_jid}, #db_mapping{column = remote_resource, param = remote_jid, format = jid_resource}, #db_mapping{column = direction, param = direction, format = direction}, - #db_mapping{column = from_jid, param = source_jid, format = jid, module = ?MODULE}, + #db_mapping{column = from_jid, param = source_jid, format = jid}, #db_mapping{column = origin_id, param = origin_id, format = maybe_binary}, - #db_mapping{column = message, param = packet, format = xml, module = ?MODULE}, - #db_mapping{column = search_body, param = packet, format = search, module = mod_mam}]. + #db_mapping{column = message, param = packet, format = xml}, + #db_mapping{column = search_body, param = packet, format = search}]. -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). prepare_message(Host, Params) -> - [prepare_value(Host, Params, Mapping) || Mapping <- db_mappings()]. + Env = env_vars(Host), + [prepare_value(Params, Env, Mapping) || Mapping <- db_mappings()]. -prepare_value(Host, Params, Mapping = #db_mapping{param = Param, format = Format}) -> +prepare_value(Params, Env, Mapping = #db_mapping{param = Param, format = Format}) -> Value = maps:get(Param, Params), - encode_value(Format, Value, Host, Params, Mapping). + encode_value(Format, Value, Params, Env). -encode_value(int, Value, _Host, _Params, _Mapping) when is_integer(Value) -> +encode_value(int, Value, _Params, _Env) when is_integer(Value) -> Value; -encode_value(maybe_binary, none, _Host, _Params, _Mapping) -> +encode_value(maybe_binary, none, _Params, _Env) -> null; -encode_value(maybe_binary, Value, _Host, _Params, _Mapping) when is_binary(Value) -> +encode_value(maybe_binary, Value, _Params, _Env) when is_binary(Value) -> Value; -encode_value(direction, Value, _Host, _Params, _Mapping) -> +encode_value(direction, Value, _Params, _Env) -> encode_direction(Value); -encode_value(bare_jid, Value, Host, #{local_jid := LocJID}, #db_mapping{module = Module}) -> - jid_to_stored_binary(Host, Module, LocJID, jid:to_bare(Value)); -encode_value(jid, Value, Host, #{local_jid := LocJID}, #db_mapping{module = Module}) -> - jid_to_stored_binary(Host, Module, LocJID, Value); -encode_value(jid_resource, #jid{lresource = Res}, _Host, _Params, _Mapping) -> +encode_value(bare_jid, Value, #{local_jid := LocJID}, #{db_jid_codec := Codec}) -> + jid_to_stored_binary_with_codec(Codec, LocJID, jid:to_bare(Value)); +encode_value(jid, Value, #{local_jid := LocJID}, #{db_jid_codec := Codec}) -> + jid_to_stored_binary_with_codec(Codec, LocJID, Value); +encode_value(jid_resource, #jid{lresource = Res}, _Params, _Env) -> Res; -encode_value(xml, Value, Host, _Params, #db_mapping{module = Module}) -> - packet_to_stored_binary(Host, Module, Value); -encode_value(search, Value, Host, _Params, #db_mapping{module = Module}) -> - mod_mam_utils:packet_to_search_body(Module, Host, Value). +encode_value(xml, Value, _Params, #{db_message_codec := Codec}) -> + packet_to_stored_binary_with_codec(Codec, Value); +encode_value(search, Value, _Params, #{full_text_search := SearchEnabled}) -> + mod_mam_utils:packet_to_search_body(SearchEnabled, Value). column_names() -> [Column || #db_mapping{column = Column} <- db_mappings()]. @@ -589,6 +589,9 @@ maybe_encode_compact_uuid(Microseconds, NodeID) -> jid_to_stored_binary(Host, Module, ArcJID, JID) -> Codec = db_jid_codec(Host, Module), + jid_to_stored_binary_with_codec(Codec, ArcJID, JID). + +jid_to_stored_binary_with_codec(Codec, ArcJID, JID) -> mam_jid:encode(Codec, ArcJID, JID). stored_binary_to_jid(Host, Module, ArcJID, BSrcJID) -> @@ -599,10 +602,18 @@ packet_to_stored_binary(Host, Module, Packet) -> Codec = db_message_codec(Host, Module), mam_message:encode(Codec, Packet). +packet_to_stored_binary_with_codec(Codec, Packet) -> + mam_message:encode(Codec, Packet). + stored_binary_to_packet(Host, Module, Bin) -> Codec = db_message_codec(Host, Module), mam_message:decode(Codec, Bin). +env_vars(Host) -> + #{full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), + db_jid_codec => db_jid_codec(Host, ?MODULE), + db_message_codec => db_message_codec(Host, ?MODULE)}. + -spec db_jid_codec(jid:server(), module()) -> module(). db_jid_codec(Host, Module) -> gen_mod:get_module_opt(Host, Module, db_jid_format, mam_jid_mini). diff --git a/src/mam/mod_mam_utils.erl b/src/mam/mod_mam_utils.erl index 1372611fbd..7394c6c7aa 100644 --- a/src/mam/mod_mam_utils.erl +++ b/src/mam/mod_mam_utils.erl @@ -59,6 +59,7 @@ normalize_search_text/1, normalize_search_text/2, packet_to_search_body/3, + packet_to_search_body/2, has_full_text_search/2 ]). @@ -761,13 +762,16 @@ normalize_search_text(Text, WordSeparator) -> -spec packet_to_search_body(Module :: mod_mam | mod_mam_muc, Host :: jid:server(), Packet :: exml:element()) -> binary(). packet_to_search_body(Module, Host, Packet) -> - case has_full_text_search(Module, Host) of - true -> - BodyValue = exml_query:path(Packet, [{element, <<"body">>}, cdata], <<>>), - mod_mam_utils:normalize_search_text(BodyValue, <<" ">>); - false -> - <<>> - end. + SearchEnabled = has_full_text_search(Module, Host), + packet_to_search_body(SearchEnabled, Packet). + +-spec packet_to_search_body(Enabled :: boolean(), + Packet :: exml:element()) -> binary(). +packet_to_search_body(true, Packet) -> + BodyValue = exml_query:path(Packet, [{element, <<"body">>}, cdata], <<>>), + mod_mam_utils:normalize_search_text(BodyValue, <<" ">>); +packet_to_search_body(false, Packet) -> + <<>>. -spec has_full_text_search(Module :: mod_mam | mod_mam_muc, Host :: jid:server()) -> boolean(). has_full_text_search(Module, Host) -> From 10a8ae5c69632ff9c1d25f30464b6b3d809bbb8b Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 20:35:02 +0100 Subject: [PATCH 15/92] Use env_vars for lookups --- src/mam/mod_mam_rdbms_arch.erl | 123 ++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 55 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 50191b8e40..c63e7b0f65 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -64,6 +64,8 @@ -type filter() :: [filter_field()]. +-type env_vars() :: map(). + %% ---------------------------------------------------------------------- %% gen_mod callbacks %% Starting and stopping functions for users' archives @@ -112,8 +114,9 @@ get_mam_pm_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> case mod_mam:archive_id(Host, User) of undefined -> []; ArchiveID -> + Env = env_vars(Host, ArcJID), {selected, Rows} = extract_gdpr_messages(Host, ArchiveID), - Messages = rows_to_uniform_format(Host, ?MODULE, ArcJID, Rows), + Messages = rows_to_uniform_format(Rows, Env), [uniform_to_gdpr(M) || M <- Messages] ++ Acc end. @@ -248,31 +251,31 @@ db_mappings() -> #db_mapping{column = search_body, param = packet, format = search}]. -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). -prepare_message(Host, Params) -> - Env = env_vars(Host), +prepare_message(Host, Params = #{local_jid := ArcJID}) -> + Env = env_vars(Host, ArcJID), [prepare_value(Params, Env, Mapping) || Mapping <- db_mappings()]. -prepare_value(Params, Env, Mapping = #db_mapping{param = Param, format = Format}) -> +prepare_value(Params, Env, #db_mapping{param = Param, format = Format}) -> Value = maps:get(Param, Params), - encode_value(Format, Value, Params, Env). + encode_value(Format, Value, Env). -encode_value(int, Value, _Params, _Env) when is_integer(Value) -> +encode_value(int, Value, _Env) when is_integer(Value) -> Value; -encode_value(maybe_binary, none, _Params, _Env) -> +encode_value(maybe_binary, none, _Env) -> null; -encode_value(maybe_binary, Value, _Params, _Env) when is_binary(Value) -> +encode_value(maybe_binary, Value, _Env) when is_binary(Value) -> Value; -encode_value(direction, Value, _Params, _Env) -> +encode_value(direction, Value, _Env) -> encode_direction(Value); -encode_value(bare_jid, Value, #{local_jid := LocJID}, #{db_jid_codec := Codec}) -> - jid_to_stored_binary_with_codec(Codec, LocJID, jid:to_bare(Value)); -encode_value(jid, Value, #{local_jid := LocJID}, #{db_jid_codec := Codec}) -> - jid_to_stored_binary_with_codec(Codec, LocJID, Value); -encode_value(jid_resource, #jid{lresource = Res}, _Params, _Env) -> +encode_value(bare_jid, Value, Env) -> + jid_to_stored_binary_with_env(jid:to_bare(Value), Env); +encode_value(jid, Value, Env) -> + jid_to_stored_binary_with_env(Value, Env); +encode_value(jid_resource, #jid{lresource = Res}, _Env) -> Res; -encode_value(xml, Value, _Params, #{db_message_codec := Codec}) -> - packet_to_stored_binary_with_codec(Codec, Value); -encode_value(search, Value, _Params, #{full_text_search := SearchEnabled}) -> +encode_value(xml, Value, Env) -> + packet_to_stored_binary_with_env(Value, Env); +encode_value(search, Value, #{full_text_search := SearchEnabled}) -> mod_mam_utils:packet_to_search_body(SearchEnabled, Value). column_names() -> @@ -308,11 +311,12 @@ extract_gdpr_messages(Host, ArchiveID) -> {ok, mod_mam:lookup_result()}. lookup_messages({error, _Reason}=Result, _Host, _Params) -> Result; -lookup_messages(_Result, Host, Params) -> +lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> try + Env = env_vars(Host, ArcJID), ExtParams = extend_params(Host, Params), Filter = prepare_filter(ExtParams), - choose_lookup_messages_strategy(Host, Filter, ExtParams) + choose_lookup_messages_strategy(Host, Env, Filter, ExtParams) catch _Type:Reason:S -> {error, {Reason, {stacktrace, S}}} end. @@ -334,32 +338,31 @@ opt_count_type(_) -> %% - we can reduce number of queries if we skip counting for small data sets; %% - sometimes we want not to count at all %% (for example, our client side counts ones and keep the information) -choose_lookup_messages_strategy(Host, Filter, - Params = #{owner_jid := ArcJID, rsm := RSM, - page_size := PageSize}) -> +choose_lookup_messages_strategy(Host, Env, Filter, + Params = #{rsm := RSM, page_size := PageSize}) -> case Params of #{is_simple := true} -> %% Simple query without calculating offset and total count - simple_lookup_messages(Host, ArcJID, RSM, PageSize, Filter); + simple_lookup_messages(Host, Env, RSM, PageSize, Filter); %% NOTICE: We always prefer opt_count optimization, if possible. %% Clients don't event know what opt_count is. #{opt_count_type := last_page} when PageSize > 0 -> %% Extract messages before calculating offset and total count %% Useful for small result sets - lookup_last_page(Host, ArcJID, PageSize, Filter); + lookup_last_page(Host, Env, PageSize, Filter); #{opt_count_type := by_offset} when PageSize > 0 -> %% Extract messages before calculating offset and total count %% Useful for small result sets - lookup_by_offset(Host, ArcJID, RSM, PageSize, Filter); + lookup_by_offset(Host, Env, RSM, PageSize, Filter); _ -> %% Calculate offset and total count first before extracting messages - lookup_messages_regular(Host, ArcJID, RSM, PageSize, Filter) + lookup_messages_regular(Host, Env, RSM, PageSize, Filter) end. %% Just extract messages -simple_lookup_messages(Host, ArcJID, RSM, PageSize, Filter) -> +simple_lookup_messages(Host, Env, RSM, PageSize, Filter) -> {Filter2, Offset, Order} = rsm_to_filter(RSM, Filter), - Messages = extract_messages(Host, ArcJID, Filter2, Offset, PageSize, Order), + Messages = extract_messages(Host, Env, Filter2, Offset, PageSize, Order), {ok, {undefined, undefined, Messages}}. rsm_to_filter(RSM, Filter) -> @@ -377,8 +380,8 @@ rsm_to_filter(RSM, Filter) -> %% This function handles case: #rsm_in{direction = before, id = undefined} %% Assumes assert_rsm_without_id(RSM) -lookup_last_page(Host, ArcJID, PageSize, Filter) -> - Messages = extract_messages(Host, ArcJID, Filter, 0, PageSize, desc), +lookup_last_page(Host, Env, PageSize, Filter) -> + Messages = extract_messages(Host, Env, Filter, 0, PageSize, desc), Selected = length(Messages), Offset = case Selected < PageSize of @@ -390,10 +393,10 @@ lookup_last_page(Host, ArcJID, PageSize, Filter) -> end, {ok, {Offset + Selected, Offset, Messages}}. -lookup_by_offset(Host, ArcJID, RSM, PageSize, Filter) -> +lookup_by_offset(Host, Env, RSM, PageSize, Filter) -> assert_rsm_without_id(RSM), Offset = rsm_to_index(RSM), - Messages = extract_messages(Host, ArcJID, Filter, Offset, PageSize, asc), + Messages = extract_messages(Host, Env, Filter, Offset, PageSize, asc), Selected = length(Messages), TotalCount = case Selected < PageSize of @@ -413,17 +416,17 @@ rsm_to_index(#rsm_in{direction = undefined, index = Offset}) when is_integer(Offset) -> Offset; rsm_to_index(_) -> 0. -lookup_messages_regular(Host, ArcJID, RSM, PageSize, Filter) -> +lookup_messages_regular(Host, Env, RSM, PageSize, Filter) -> TotalCount = calc_count(Host, Filter), Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), Messages = case RSM of #rsm_in{direction = aft, id = ID} -> - extract_messages(Host, ArcJID, from_id(ID, Filter), 0, PageSize + 1, asc); + extract_messages(Host, Env, from_id(ID, Filter), 0, PageSize + 1, asc); #rsm_in{direction = before, id = ID} when ID =/= undefined -> - extract_messages(Host, ArcJID, to_id(ID, Filter), 0, PageSize + 1, desc); + extract_messages(Host, Env, to_id(ID, Filter), 0, PageSize + 1, desc); _ -> - extract_messages(Host, ArcJID, Filter, Offset, PageSize, asc) + extract_messages(Host, Env, Filter, Offset, PageSize, asc) end, Result = {TotalCount, Offset, Messages}, mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result). @@ -447,14 +450,14 @@ from_id(ID, Filter) -> to_id(ID, Filter) -> [{le, id, ID}|Filter]. -rows_to_uniform_format(Host, Module, ArcJID, MessageRows) -> - [row_to_uniform_format(Host, Module, ArcJID, Row) || Row <- MessageRows]. +rows_to_uniform_format(MessageRows, Env) -> + [row_to_uniform_format(Row, Env) || Row <- MessageRows]. -row_to_uniform_format(Host, Module, ArcJID, {BMessID, BSrcJID, SDataRaw}) -> +row_to_uniform_format({BMessID, BSrcJID, SDataRaw}, Env) -> MessID = mongoose_rdbms:result_to_integer(BMessID), - SrcJID = stored_binary_to_jid(Host, Module, ArcJID, BSrcJID), - Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Packet = stored_binary_to_packet(Host, Module, Data), + SrcJID = stored_binary_to_jid_with_env(BSrcJID, Env), + Data = unescape_binary_with_env(SDataRaw, Env), + Packet = stored_binary_to_packet_with_env(Data, Env), {MessID, SrcJID, Packet}. uniform_to_message_id({MessID, _, _}) -> MessID. @@ -462,18 +465,18 @@ uniform_to_message_id({MessID, _, _}) -> MessID. %% @doc Each record is a tuple of form %% `{<<"13663125233">>, <<"bob@localhost">>, <>}'. %% Columns are `["id", "from_jid", "message"]'. --spec extract_messages(Host :: jid:server(), ArcJID :: jid:jid(), +-spec extract_messages(Host :: jid:server(), Env :: env_vars(), Filter :: filter(), Offset :: non_neg_integer(), Max :: pos_integer(), Order :: asc | desc) -> [mod_mam:message_row()]. -extract_messages(_Host, _ArcJID, _Filter, _Offset, 0, _) -> +extract_messages(_Host, _Env, _Filter, _Offset, 0 = _Max, _Order) -> []; -extract_messages(Host, ArcJID, Filter, Offset, Max, Order) -> +extract_messages(Host, Env, Filter, Offset, Max, Order) -> {selected, MessageRows} = extract_rows(Host, Filter, Offset, Max, Order), ?LOG_DEBUG(#{what => mam_extract_messages, mam_filter => Filter, offset => Offset, max => Max, - host => Host, message_rows => MessageRows}), + host => Host, env_vars => Env, message_rows => MessageRows}), Rows = maybe_reverse(Order, MessageRows), - rows_to_uniform_format(Host, ?MODULE, ArcJID, Rows). + rows_to_uniform_format(Rows, Env). extract_rows(Host, Filters, Offset, Max, Order) -> Filters2 = Filters ++ rdbms_queries:limit_offset_filters(Max, Offset), @@ -589,28 +592,38 @@ maybe_encode_compact_uuid(Microseconds, NodeID) -> jid_to_stored_binary(Host, Module, ArcJID, JID) -> Codec = db_jid_codec(Host, Module), - jid_to_stored_binary_with_codec(Codec, ArcJID, JID). + mam_jid:encode(Codec, ArcJID, JID). -jid_to_stored_binary_with_codec(Codec, ArcJID, JID) -> +jid_to_stored_binary_with_env(JID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> mam_jid:encode(Codec, ArcJID, JID). -stored_binary_to_jid(Host, Module, ArcJID, BSrcJID) -> - Codec = db_jid_codec(Host, Module), - mam_jid:decode(Codec, ArcJID, BSrcJID). +stored_binary_to_jid_with_env(EncodedJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> + mam_jid:decode(Codec, ArcJID, EncodedJID). packet_to_stored_binary(Host, Module, Packet) -> Codec = db_message_codec(Host, Module), mam_message:encode(Codec, Packet). -packet_to_stored_binary_with_codec(Codec, Packet) -> +packet_to_stored_binary_with_env(Packet, #{db_message_codec := Codec}) -> mam_message:encode(Codec, Packet). stored_binary_to_packet(Host, Module, Bin) -> Codec = db_message_codec(Host, Module), mam_message:decode(Codec, Bin). -env_vars(Host) -> - #{full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), +stored_binary_to_packet_with_env(Bin, #{db_message_codec := Codec}) -> + mam_message:decode(Codec, Bin). + +unescape_binary_with_env(Bin, #{host := Host}) -> + %% Funny, rdbms ignores this Host variable + mongoose_rdbms:unescape_binary(Host, Bin). + +env_vars(Host, ArcJID) -> + %% Please, minimize usage of the host field. + %% It's only for passing into RDBMS. + #{host => Host, + archive_jid => ArcJID, + full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), db_jid_codec => db_jid_codec(Host, ?MODULE), db_message_codec => db_message_codec(Host, ?MODULE)}. From d75bfebf6041f05fe25cd2536c6d15975bcc1dc7 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 21:00:57 +0100 Subject: [PATCH 16/92] Use env_vars for retraction --- src/mam/mod_mam_rdbms_arch.erl | 81 ++++++++++++++++------------------ src/mam/mod_mam_utils.erl | 12 +++-- 2 files changed, 47 insertions(+), 46 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index c63e7b0f65..031b72108d 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -180,9 +180,10 @@ index_hint_sql(Host) -> -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok. -archive_message(_Result, Host, Params) -> +archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> try - do_archive_message(Host, Params) + Env = env_vars(Host, ArcJID), + do_archive_message(Host, Params, Env) catch Class:Reason:StackTrace -> ?LOG_ERROR(#{what => archive_message_failed, host => Host, mam_params => Params, @@ -190,41 +191,45 @@ archive_message(_Result, Host, Params) -> {error, Reason} end. -do_archive_message(Host, Params) -> - Row = prepare_message(Host, Params), +do_archive_message(Host, Params, Env) -> + Row = prepare_message_with_env(Params, Env), {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_message, Row), - retract_message(Host, Params). + retract_message(Host, Params, Env). %% Retraction logic -spec retract_message(jid:server(), mod_mam:archive_message_params()) -> ok. +retract_message(Host, #{local_jid := ArcJID} = Params) -> + Env = env_vars(Host, ArcJID), + retract_message(Host, Params, Env). + +-spec retract_message(jid:server(), mod_mam:archive_message_params(), env_vars()) -> ok. retract_message(Host, #{archive_id := UserID, - local_jid := LocJID, remote_jid := RemJID, direction := Dir, - packet := Packet}) -> - case mod_mam_utils:get_retract_id(mod_mam, Host, Packet) of + packet := Packet}, Env) -> + case get_retract_id_with_env(Packet, Env) of none -> ok; - OriginIDToRetract -> retract_message(Host, UserID, LocJID, RemJID, OriginIDToRetract, Dir) + OriginIDToRetract -> retract_message(Host, UserID, RemJID, OriginIDToRetract, Dir, Env) end. -retract_message(Host, UserID, LocJID, RemJID, OriginID, Dir) -> - MinBareRemJID = minify_bare_jid(Host, ?MODULE, LocJID, RemJID), +retract_message(Host, UserID, RemJID, OriginID, Dir, Env) -> + MinBareRemJID = jid_to_stored_binary_with_env(jid:to_bare(RemJID), Env), BinDir = encode_direction(Dir), {selected, Rows} = execute_select_messages_to_retract( Host, UserID, MinBareRemJID, OriginID, BinDir), - make_tombstone(Host, UserID, OriginID, Rows), + make_tombstone(Host, UserID, OriginID, Rows, Env), ok. -make_tombstone(_Host, UserID, OriginID, []) -> +make_tombstone(_Host, UserID, OriginID, [], _Env) -> ?LOG_INFO(#{what => make_tombstone_failed, text => <<"Message to retract was not found by origin id">>, user_id => UserID, origin_id => OriginID}); -make_tombstone(Host, UserID, OriginID, [{ResMessID, ResData}]) -> - Data = mongoose_rdbms:unescape_binary(Host, ResData), - Packet = stored_binary_to_packet(Host, ?MODULE, Data), +make_tombstone(Host, UserID, OriginID, [{ResMessID, ResData}], Env) -> + Data = unescape_binary_with_env(ResData, Env), + Packet = stored_binary_to_packet_with_env(Data, Env), MessID = mongoose_rdbms:result_to_integer(ResMessID), Tombstone = mod_mam_utils:tombstone(Packet, OriginID), - TombstoneData = packet_to_stored_binary(Host, ?MODULE, Tombstone), + TombstoneData = packet_to_stored_binary_with_env(Tombstone, Env), execute_make_tombstone(Host, TombstoneData, UserID, MessID). execute_select_messages_to_retract(Host, UserID, BareRemJID, OriginID, Dir) -> @@ -253,6 +258,9 @@ db_mappings() -> -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). prepare_message(Host, Params = #{local_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), + prepare_message_with_env(Params, Env). + +prepare_message_with_env(Params, Env) -> [prepare_value(Params, Env, Mapping) || Mapping <- db_mappings()]. prepare_value(Params, Env, #db_mapping{param = Param, format = Format}) -> @@ -314,7 +322,7 @@ lookup_messages({error, _Reason}=Result, _Host, _Params) -> lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> try Env = env_vars(Host, ArcJID), - ExtParams = extend_params(Host, Params), + ExtParams = extend_params(Params, Env), Filter = prepare_filter(ExtParams), choose_lookup_messages_strategy(Host, Env, Filter, ExtParams) catch _Type:Reason:S -> @@ -567,15 +575,7 @@ maybe_reverse(desc, List) -> lists:reverse(List). %% ---------------------------------------------------------------------- -%% Optimizations - -maybe_minify_bare_jid(_Host, _Module, _LocJID, undefined) -> - undefined; -maybe_minify_bare_jid(Host, Module, LocJID, JID) -> - minify_bare_jid(Host, Module, LocJID, JID). - -minify_bare_jid(Host, Module, LocJID, JID) -> - jid_to_stored_binary(Host, Module, LocJID, jid:to_bare(JID)). +%% Optimizations and extensible code make_start_id(Start, Borders) -> StartID = maybe_encode_compact_uuid(Start, 0), @@ -590,9 +590,10 @@ maybe_encode_compact_uuid(undefined, _) -> maybe_encode_compact_uuid(Microseconds, NodeID) -> encode_compact_uuid(Microseconds, NodeID). -jid_to_stored_binary(Host, Module, ArcJID, JID) -> - Codec = db_jid_codec(Host, Module), - mam_jid:encode(Codec, ArcJID, JID). +maybe_minify_bare_jid(undefined, _Env) -> + undefined; +maybe_minify_bare_jid(JID, Env) -> + jid_to_stored_binary_with_env(jid:to_bare(JID), Env). jid_to_stored_binary_with_env(JID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> mam_jid:encode(Codec, ArcJID, JID). @@ -600,17 +601,9 @@ jid_to_stored_binary_with_env(JID, #{db_jid_codec := Codec, archive_jid := ArcJI stored_binary_to_jid_with_env(EncodedJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> mam_jid:decode(Codec, ArcJID, EncodedJID). -packet_to_stored_binary(Host, Module, Packet) -> - Codec = db_message_codec(Host, Module), - mam_message:encode(Codec, Packet). - packet_to_stored_binary_with_env(Packet, #{db_message_codec := Codec}) -> mam_message:encode(Codec, Packet). -stored_binary_to_packet(Host, Module, Bin) -> - Codec = db_message_codec(Host, Module), - mam_message:decode(Codec, Bin). - stored_binary_to_packet_with_env(Bin, #{db_message_codec := Codec}) -> mam_message:decode(Codec, Bin). @@ -618,11 +611,15 @@ unescape_binary_with_env(Bin, #{host := Host}) -> %% Funny, rdbms ignores this Host variable mongoose_rdbms:unescape_binary(Host, Bin). +get_retract_id_with_env(Packet, #{has_message_retraction := Enabled}) -> + mod_mam_utils:get_retract_id(Enabled, Packet). + env_vars(Host, ArcJID) -> %% Please, minimize usage of the host field. %% It's only for passing into RDBMS. #{host => Host, archive_jid => ArcJID, + has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam, Host), full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), db_jid_codec => db_jid_codec(Host, ?MODULE), db_message_codec => db_message_codec(Host, ?MODULE)}. @@ -728,14 +725,14 @@ filter_to_sql(equal, Column) -> filter_to_sql(like, Column) -> Column ++ " LIKE ?". -extend_params(Host, #{owner_jid := ArcJID, rsm := RSM, borders := Borders, - start_ts := Start, end_ts := End, with_jid := WithJID, - search_text := SearchText} = Params) -> +extend_params(#{rsm := RSM, borders := Borders, + start_ts := Start, end_ts := End, with_jid := WithJID, + search_text := SearchText} = Params, Env) -> Params#{opt_count_type => opt_count_type(RSM), norm_search_text => mod_mam_utils:normalize_search_text(SearchText), start_id => make_start_id(Start, Borders), end_id => make_end_id(End, Borders), - remote_bare_jid => maybe_minify_bare_jid(Host, ?MODULE, ArcJID, WithJID), + remote_bare_jid => maybe_minify_bare_jid(WithJID, Env), remote_resource => jid_to_non_empty_resource(WithJID)}. jid_to_non_empty_resource(#jid{lresource = Res}) when byte_size(Res) > 0 -> diff --git a/src/mam/mod_mam_utils.erl b/src/mam/mod_mam_utils.erl index 7394c6c7aa..b30eba9582 100644 --- a/src/mam/mod_mam_utils.erl +++ b/src/mam/mod_mam_utils.erl @@ -29,6 +29,8 @@ get_one_of_path/2, get_one_of_path/3, is_archivable_message/4, + has_message_retraction/2, + get_retract_id/2, get_retract_id/3, get_origin_id/1, tombstone/2, @@ -357,10 +359,12 @@ has_chat_marker(Packet) -> end. get_retract_id(Module, Host, Packet) -> - case has_message_retraction(Module, Host) of - true -> get_retract_id(Packet); - false -> none - end. + get_retract_id(has_message_retraction(Module, Host), Packet). + +get_retract_id(true = _Enabled, Packet) -> + get_retract_id(Packet); +get_retract_id(false, Packet) -> + none. get_retract_id(Packet) -> case exml_query:subelement_with_name_and_ns(Packet, <<"apply-to">>, ?NS_FASTEN) of From 6facbf86eae224b86448c402172b2f769f8bedb5 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 21:21:57 +0100 Subject: [PATCH 17/92] Use generic lookup for GDPR data extraction in mod_mam --- src/mam/mod_mam_rdbms_arch.erl | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 031b72108d..9b45b5b5ae 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -94,13 +94,6 @@ start(Host, Opts) -> " WHERE user_id = ? AND remote_bare_jid = ? " " AND origin_id = ? AND direction = ?" " ORDER BY id DESC ", LimitSQL/binary>>]), - - mongoose_rdbms:prepare(mam_extract_gdpr_messages, mam_message, - [user_id], - [<<"SELECT id, from_jid, message " - " FROM mam_message " - " WHERE user_id=? " - " ORDER BY id">>]), ok. @@ -311,8 +304,8 @@ remove_archive(Host, UserID) -> %% GDPR logic extract_gdpr_messages(Host, ArchiveID) -> - mod_mam_utils:success_sql_execute(Host, mam_extract_gdpr_messages, [ArchiveID]). - + Filters = [{user_id, ArchiveID}], + lookup_query(lookup, Host, Filters, asc). %% Lookup logic -spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> @@ -639,7 +632,7 @@ lookup_sql_binary(QueryType, Filters, Order, IndexHintSQL) -> iolist_to_binary(lookup_sql(QueryType, Filters, Order, IndexHintSQL)). lookup_sql(QueryType, Filters, Order, IndexHintSQL) -> - LimitSQL = limit_sql(QueryType), + LimitSQL = limit_sql(Filters), OrderSQL = order_to_sql(Order), FilterSQL = filters_to_sql(Filters), ["SELECT ", columns_sql(QueryType), " " @@ -649,8 +642,14 @@ lookup_sql(QueryType, Filters, Order, IndexHintSQL) -> columns_sql(lookup) -> "id, from_jid, message"; columns_sql(count) -> "COUNT(*)". -limit_sql(lookup) -> rdbms_queries:limit_offset_sql(); -limit_sql(count) -> "". +%% Caller should provide both limit and offset fields in the correct order. +%% See limit_offset_filters. +%% No limits option is fine too (it is used with count and GDPR). +limit_sql(Filters) -> + case lists:keymember(limit, 1, Filters) of %% and offset + true -> rdbms_queries:limit_offset_sql(); + false -> "" + end. filters_to_columns(Filters) -> [Column || {_Op, Column, _Value} <- Filters]. From a610e552930d934aba799bfd219ed6f3a837e403 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 21:36:37 +0100 Subject: [PATCH 18/92] Shorten calc_count specs --- src/mam/mod_mam_rdbms_arch.erl | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 9b45b5b5ae..2d9e6c70de 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -371,6 +371,8 @@ rsm_to_filter(RSM, Filter) -> %% Get last rows from result set #rsm_in{direction = aft, id = ID} -> {after_id(ID, Filter), 0, asc}; + #rsm_in{direction = before, id = undefined} -> + {Filter, 0, desc}; #rsm_in{direction = before, id = ID} -> {before_id(ID, Filter), 0, desc}; #rsm_in{direction = undefined, index = Index} -> @@ -432,22 +434,19 @@ lookup_messages_regular(Host, Env, RSM, PageSize, Filter) -> Result = {TotalCount, Offset, Messages}, mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result). --spec after_id(ID :: mod_mam:message_id(), Filter :: filter()) -> filter(). +-spec after_id(mod_mam:message_id(), filter()) -> filter(). after_id(ID, Filter) -> [{greater, id, ID}|Filter]. --spec before_id(ID :: mod_mam:message_id() | undefined, - Filter :: filter()) -> filter(). -before_id(undefined, Filter) -> - Filter; +-spec before_id(mod_mam:message_id(), filter()) -> filter(). before_id(ID, Filter) -> [{lower, id, ID}|Filter]. --spec from_id(ID :: mod_mam:message_id(), Filter :: filter()) -> filter(). +-spec from_id(mod_mam:message_id(), filter()) -> filter(). from_id(ID, Filter) -> [{ge, id, ID}|Filter]. --spec to_id(ID :: mod_mam:message_id(), Filter :: filter()) -> filter(). +-spec to_id(mod_mam:message_id(), filter()) -> filter(). to_id(ID, Filter) -> [{le, id, ID}|Filter]. @@ -519,27 +518,22 @@ lookup_query(QueryType, Host, Filters, Order) -> %% be returned instead. %% @end %% "SELECT COUNT(*) as "index" FROM mam_message WHERE id <= '", UID --spec calc_index(Host :: jid:server(), - Filter :: filter(), - ID :: mod_mam:message_id()) -> non_neg_integer(). +-spec calc_index(jid:server(), filter(), mod_mam:message_id()) -> non_neg_integer(). calc_index(Host, Filter, ID) -> - calc_count(Host, [{le, id, ID}|Filter]). + calc_count(Host, to_id(ID, Filter)). %% @doc Count of elements in RSet before the passed element. %% %% The element with the passed UID can be already deleted. %% @end %% "SELECT COUNT(*) as "count" FROM mam_message WHERE id < '", UID --spec calc_before(Host :: jid:server(), - Filter :: filter(), - ID :: mod_mam:message_id()) -> non_neg_integer(). +-spec calc_before(jid:server(), filter(), mod_mam:message_id()) -> non_neg_integer(). calc_before(Host, Filter, ID) -> - calc_count(Host, [{lower, id, ID}|Filter]). + calc_count(Host, before_id(ID, Filter)). %% @doc Get the total result set size. %% "SELECT COUNT(*) as "count" FROM mam_message WHERE " --spec calc_count(Host :: jid:server(), - Filter :: filter()) -> non_neg_integer(). +-spec calc_count(jid:server(), filter()) -> non_neg_integer(). calc_count(Host, Filter) -> Result = lookup_query(count, Host, Filter, unordered), mongoose_rdbms:selected_to_integer(Result). From 3fc2890ffed4529fca199dc3c4017a6007fac10e Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 21:48:29 +0100 Subject: [PATCH 19/92] Add specs for with_env functions --- src/mam/mod_mam_rdbms_arch.erl | 51 ++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 2d9e6c70de..b01fee64c2 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -185,7 +185,7 @@ archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> end. do_archive_message(Host, Params, Env) -> - Row = prepare_message_with_env(Params, Env), + Row = prepare_message(Params, Env), {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_message, Row), retract_message(Host, Params, Env). @@ -200,13 +200,13 @@ retract_message(Host, #{archive_id := UserID, remote_jid := RemJID, direction := Dir, packet := Packet}, Env) -> - case get_retract_id_with_env(Packet, Env) of + case get_retract_id(Packet, Env) of none -> ok; OriginIDToRetract -> retract_message(Host, UserID, RemJID, OriginIDToRetract, Dir, Env) end. retract_message(Host, UserID, RemJID, OriginID, Dir, Env) -> - MinBareRemJID = jid_to_stored_binary_with_env(jid:to_bare(RemJID), Env), + MinBareRemJID = encode_jid(jid:to_bare(RemJID), Env), BinDir = encode_direction(Dir), {selected, Rows} = execute_select_messages_to_retract( Host, UserID, MinBareRemJID, OriginID, BinDir), @@ -218,11 +218,11 @@ make_tombstone(_Host, UserID, OriginID, [], _Env) -> text => <<"Message to retract was not found by origin id">>, user_id => UserID, origin_id => OriginID}); make_tombstone(Host, UserID, OriginID, [{ResMessID, ResData}], Env) -> - Data = unescape_binary_with_env(ResData, Env), - Packet = stored_binary_to_packet_with_env(Data, Env), + Data = unescape_binary(ResData, Env), + Packet = decode_packet(Data, Env), MessID = mongoose_rdbms:result_to_integer(ResMessID), Tombstone = mod_mam_utils:tombstone(Packet, OriginID), - TombstoneData = packet_to_stored_binary_with_env(Tombstone, Env), + TombstoneData = encode_packet(Tombstone, Env), execute_make_tombstone(Host, TombstoneData, UserID, MessID). execute_select_messages_to_retract(Host, UserID, BareRemJID, OriginID, Dir) -> @@ -251,9 +251,9 @@ db_mappings() -> -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). prepare_message(Host, Params = #{local_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), - prepare_message_with_env(Params, Env). + prepare_message(Params, Env). -prepare_message_with_env(Params, Env) -> +prepare_message(Params, Env) -> [prepare_value(Params, Env, Mapping) || Mapping <- db_mappings()]. prepare_value(Params, Env, #db_mapping{param = Param, format = Format}) -> @@ -269,13 +269,13 @@ encode_value(maybe_binary, Value, _Env) when is_binary(Value) -> encode_value(direction, Value, _Env) -> encode_direction(Value); encode_value(bare_jid, Value, Env) -> - jid_to_stored_binary_with_env(jid:to_bare(Value), Env); + encode_jid(jid:to_bare(Value), Env); encode_value(jid, Value, Env) -> - jid_to_stored_binary_with_env(Value, Env); + encode_jid(Value, Env); encode_value(jid_resource, #jid{lresource = Res}, _Env) -> Res; encode_value(xml, Value, Env) -> - packet_to_stored_binary_with_env(Value, Env); + encode_packet(Value, Env); encode_value(search, Value, #{full_text_search := SearchEnabled}) -> mod_mam_utils:packet_to_search_body(SearchEnabled, Value). @@ -455,16 +455,13 @@ rows_to_uniform_format(MessageRows, Env) -> row_to_uniform_format({BMessID, BSrcJID, SDataRaw}, Env) -> MessID = mongoose_rdbms:result_to_integer(BMessID), - SrcJID = stored_binary_to_jid_with_env(BSrcJID, Env), - Data = unescape_binary_with_env(SDataRaw, Env), - Packet = stored_binary_to_packet_with_env(Data, Env), + SrcJID = decode_jid(BSrcJID, Env), + Data = unescape_binary(SDataRaw, Env), + Packet = decode_packet(Data, Env), {MessID, SrcJID, Packet}. uniform_to_message_id({MessID, _, _}) -> MessID. -%% @doc Each record is a tuple of form -%% `{<<"13663125233">>, <<"bob@localhost">>, <>}'. -%% Columns are `["id", "from_jid", "message"]'. -spec extract_messages(Host :: jid:server(), Env :: env_vars(), Filter :: filter(), Offset :: non_neg_integer(), Max :: pos_integer(), Order :: asc | desc) -> [mod_mam:message_row()]. @@ -580,25 +577,31 @@ maybe_encode_compact_uuid(Microseconds, NodeID) -> maybe_minify_bare_jid(undefined, _Env) -> undefined; maybe_minify_bare_jid(JID, Env) -> - jid_to_stored_binary_with_env(jid:to_bare(JID), Env). + encode_jid(jid:to_bare(JID), Env). -jid_to_stored_binary_with_env(JID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> +-spec encode_jid(jid:jid(), env_vars()) -> binary(). +encode_jid(JID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> mam_jid:encode(Codec, ArcJID, JID). -stored_binary_to_jid_with_env(EncodedJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> +-spec decode_jid(binary(), env_vars()) -> jid:jid(). +decode_jid(EncodedJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> mam_jid:decode(Codec, ArcJID, EncodedJID). -packet_to_stored_binary_with_env(Packet, #{db_message_codec := Codec}) -> +-spec encode_packet(exml:element(), env_vars()) -> binary(). +encode_packet(Packet, #{db_message_codec := Codec}) -> mam_message:encode(Codec, Packet). -stored_binary_to_packet_with_env(Bin, #{db_message_codec := Codec}) -> +-spec encode_packet(binary(), env_vars()) -> exml:element(). +decode_packet(Bin, #{db_message_codec := Codec}) -> mam_message:decode(Codec, Bin). -unescape_binary_with_env(Bin, #{host := Host}) -> +-spec unescape_binary(binary(), env_vars()) -> binary(). +unescape_binary(Bin, #{host := Host}) -> %% Funny, rdbms ignores this Host variable mongoose_rdbms:unescape_binary(Host, Bin). -get_retract_id_with_env(Packet, #{has_message_retraction := Enabled}) -> +-spec get_retract_id(exml:element(), env_vars()) -> none | binary(). +get_retract_id(Packet, #{has_message_retraction := Enabled}) -> mod_mam_utils:get_retract_id(Enabled, Packet). env_vars(Host, ArcJID) -> From 0873a4a5c2b97cd0646b3284e29eda88379321cf Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 1 Dec 2020 22:05:24 +0100 Subject: [PATCH 20/92] Use lookup_query for archive_size --- src/mam/mod_mam_rdbms_arch.erl | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index b01fee64c2..58ab927299 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -75,10 +75,6 @@ start(Host, Opts) -> start_pm(Host, Opts), prepare_insert(insert_mam_message, 1), - mongoose_rdbms:prepare(mam_archive_size, mam_message, [user_id], - [<<"SELECT COUNT(*) FROM mam_message ">>, - index_hint_sql(Host), - <<"WHERE user_id = ?">>]), mongoose_rdbms:prepare(mam_archive_remove, mam_message, [user_id], [<<"DELETE FROM mam_message " "WHERE user_id = ?">>]), @@ -158,8 +154,8 @@ encode_direction(outgoing) -> <<"O">>. -spec archive_size(Size :: integer(), Host :: jid:server(), ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). archive_size(Size, Host, UserID, _ArcJID) when is_integer(Size) -> - Result = mod_mam_utils:success_sql_execute(Host, mam_archive_size, [UserID]), - mongoose_rdbms:selected_to_integer(Result). + Filter = [{equal, user_id, UserID}], + calc_count(Host, Filter). -spec index_hint_sql(jid:server()) -> string(). @@ -185,7 +181,7 @@ archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> end. do_archive_message(Host, Params, Env) -> - Row = prepare_message(Params, Env), + Row = prepare_message_with_env(Params, Env), {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_message, Row), retract_message(Host, Params, Env). @@ -251,9 +247,9 @@ db_mappings() -> -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). prepare_message(Host, Params = #{local_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), - prepare_message(Params, Env). + prepare_message_with_env(Params, Env). -prepare_message(Params, Env) -> +prepare_message_with_env(Params, Env) -> [prepare_value(Params, Env, Mapping) || Mapping <- db_mappings()]. prepare_value(Params, Env, #db_mapping{param = Param, format = Format}) -> @@ -304,7 +300,7 @@ remove_archive(Host, UserID) -> %% GDPR logic extract_gdpr_messages(Host, ArchiveID) -> - Filters = [{user_id, ArchiveID}], + Filters = [{equal, user_id, ArchiveID}], lookup_query(lookup, Host, Filters, asc). %% Lookup logic @@ -591,7 +587,7 @@ decode_jid(EncodedJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> encode_packet(Packet, #{db_message_codec := Codec}) -> mam_message:encode(Codec, Packet). --spec encode_packet(binary(), env_vars()) -> exml:element(). +-spec decode_packet(binary(), env_vars()) -> exml:element(). decode_packet(Bin, #{db_message_codec := Codec}) -> mam_message:decode(Codec, Bin). From 1e95f34f1df44b6aab1bdde819b80a0c6b85d6d3 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 09:43:10 +0100 Subject: [PATCH 21/92] Pass env into lookup_query --- src/mam/mod_mam_rdbms_arch.erl | 116 +++++++++++++++++---------------- 1 file changed, 59 insertions(+), 57 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 58ab927299..dd6344fddf 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -155,17 +155,8 @@ encode_direction(outgoing) -> <<"O">>. ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). archive_size(Size, Host, UserID, _ArcJID) when is_integer(Size) -> Filter = [{equal, user_id, UserID}], - calc_count(Host, Filter). - - --spec index_hint_sql(jid:server()) -> string(). -index_hint_sql(Host) -> - case mongoose_rdbms:db_engine(Host) of - mysql -> - "USE INDEX(PRIMARY, i_mam_message_rem) "; - _ -> - "" - end. + Env = #{host => Host}, + calc_count(Env, Filter). -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok. @@ -301,7 +292,8 @@ remove_archive(Host, UserID) -> %% GDPR logic extract_gdpr_messages(Host, ArchiveID) -> Filters = [{equal, user_id, ArchiveID}], - lookup_query(lookup, Host, Filters, asc). + Env = #{host => Host}, + lookup_query(lookup, Env, Filters, asc). %% Lookup logic -spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> @@ -313,7 +305,7 @@ lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), ExtParams = extend_params(Params, Env), Filter = prepare_filter(ExtParams), - choose_lookup_messages_strategy(Host, Env, Filter, ExtParams) + choose_lookup_messages_strategy(Env, Filter, ExtParams) catch _Type:Reason:S -> {error, {Reason, {stacktrace, S}}} end. @@ -335,31 +327,31 @@ opt_count_type(_) -> %% - we can reduce number of queries if we skip counting for small data sets; %% - sometimes we want not to count at all %% (for example, our client side counts ones and keep the information) -choose_lookup_messages_strategy(Host, Env, Filter, +choose_lookup_messages_strategy(Env, Filter, Params = #{rsm := RSM, page_size := PageSize}) -> case Params of #{is_simple := true} -> %% Simple query without calculating offset and total count - simple_lookup_messages(Host, Env, RSM, PageSize, Filter); + simple_lookup_messages(Env, RSM, PageSize, Filter); %% NOTICE: We always prefer opt_count optimization, if possible. %% Clients don't event know what opt_count is. #{opt_count_type := last_page} when PageSize > 0 -> %% Extract messages before calculating offset and total count %% Useful for small result sets - lookup_last_page(Host, Env, PageSize, Filter); + lookup_last_page(Env, PageSize, Filter); #{opt_count_type := by_offset} when PageSize > 0 -> %% Extract messages before calculating offset and total count %% Useful for small result sets - lookup_by_offset(Host, Env, RSM, PageSize, Filter); + lookup_by_offset(Env, RSM, PageSize, Filter); _ -> %% Calculate offset and total count first before extracting messages - lookup_messages_regular(Host, Env, RSM, PageSize, Filter) + lookup_messages_regular(Env, RSM, PageSize, Filter) end. %% Just extract messages -simple_lookup_messages(Host, Env, RSM, PageSize, Filter) -> +simple_lookup_messages(Env, RSM, PageSize, Filter) -> {Filter2, Offset, Order} = rsm_to_filter(RSM, Filter), - Messages = extract_messages(Host, Env, Filter2, Offset, PageSize, Order), + Messages = extract_messages(Env, Filter2, Offset, PageSize, Order), {ok, {undefined, undefined, Messages}}. rsm_to_filter(RSM, Filter) -> @@ -379,8 +371,8 @@ rsm_to_filter(RSM, Filter) -> %% This function handles case: #rsm_in{direction = before, id = undefined} %% Assumes assert_rsm_without_id(RSM) -lookup_last_page(Host, Env, PageSize, Filter) -> - Messages = extract_messages(Host, Env, Filter, 0, PageSize, desc), +lookup_last_page(Env, PageSize, Filter) -> + Messages = extract_messages(Env, Filter, 0, PageSize, desc), Selected = length(Messages), Offset = case Selected < PageSize of @@ -388,14 +380,14 @@ lookup_last_page(Host, Env, PageSize, Filter) -> 0; %% Result fits on a single page false -> FirstID = uniform_to_message_id(hd(Messages)), - calc_count(Host, before_id(FirstID, Filter)) + calc_count(Env, before_id(FirstID, Filter)) end, {ok, {Offset + Selected, Offset, Messages}}. -lookup_by_offset(Host, Env, RSM, PageSize, Filter) -> +lookup_by_offset(Env, RSM, PageSize, Filter) -> assert_rsm_without_id(RSM), Offset = rsm_to_index(RSM), - Messages = extract_messages(Host, Env, Filter, Offset, PageSize, asc), + Messages = extract_messages(Env, Filter, Offset, PageSize, asc), Selected = length(Messages), TotalCount = case Selected < PageSize of @@ -403,7 +395,7 @@ lookup_by_offset(Host, Env, RSM, PageSize, Filter) -> Offset + Selected; %% Result fits on a single page false -> LastID = uniform_to_message_id(lists:last(Messages)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), + CountAfterLastID = calc_count(Env, after_id(LastID, Filter)), Offset + Selected + CountAfterLastID end, {ok, {TotalCount, Offset, Messages}}. @@ -415,17 +407,17 @@ rsm_to_index(#rsm_in{direction = undefined, index = Offset}) when is_integer(Offset) -> Offset; rsm_to_index(_) -> 0. -lookup_messages_regular(Host, Env, RSM, PageSize, Filter) -> - TotalCount = calc_count(Host, Filter), - Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), +lookup_messages_regular(Env, RSM, PageSize, Filter) -> + TotalCount = calc_count(Env, Filter), + Offset = calc_offset(Env, Filter, PageSize, TotalCount, RSM), Messages = case RSM of #rsm_in{direction = aft, id = ID} -> - extract_messages(Host, Env, from_id(ID, Filter), 0, PageSize + 1, asc); + extract_messages(Env, from_id(ID, Filter), 0, PageSize + 1, asc); #rsm_in{direction = before, id = ID} when ID =/= undefined -> - extract_messages(Host, Env, to_id(ID, Filter), 0, PageSize + 1, desc); + extract_messages(Env, to_id(ID, Filter), 0, PageSize + 1, desc); _ -> - extract_messages(Host, Env, Filter, Offset, PageSize, asc) + extract_messages(Env, Filter, Offset, PageSize, asc) end, Result = {TotalCount, Offset, Messages}, mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result). @@ -458,24 +450,32 @@ row_to_uniform_format({BMessID, BSrcJID, SDataRaw}, Env) -> uniform_to_message_id({MessID, _, _}) -> MessID. --spec extract_messages(Host :: jid:server(), Env :: env_vars(), +-spec extract_messages(Env :: env_vars(), Filter :: filter(), Offset :: non_neg_integer(), Max :: pos_integer(), Order :: asc | desc) -> [mod_mam:message_row()]. -extract_messages(_Host, _Env, _Filter, _Offset, 0 = _Max, _Order) -> +extract_messages(_Env, _Filter, _Offset, 0 = _Max, _Order) -> []; -extract_messages(Host, Env, Filter, Offset, Max, Order) -> - {selected, MessageRows} = extract_rows(Host, Filter, Offset, Max, Order), +extract_messages(Env, Filter, Offset, Max, Order) -> + {selected, MessageRows} = extract_rows(Env, Filter, Offset, Max, Order), ?LOG_DEBUG(#{what => mam_extract_messages, mam_filter => Filter, offset => Offset, max => Max, - host => Host, env_vars => Env, message_rows => MessageRows}), + env_vars => Env, message_rows => MessageRows}), Rows = maybe_reverse(Order, MessageRows), rows_to_uniform_format(Rows, Env). -extract_rows(Host, Filters, Offset, Max, Order) -> +extract_rows(Env, Filters, Offset, Max, Order) -> Filters2 = Filters ++ rdbms_queries:limit_offset_filters(Max, Offset), - lookup_query(lookup, Host, Filters2, Order). + lookup_query(lookup, Env, Filters2, Order). + +%% @doc Get the total result set size. +%% "SELECT COUNT(*) as "count" FROM mam_message WHERE " +-spec calc_count(env_vars(), filter()) -> non_neg_integer(). +calc_count(Env, Filter) -> + Result = lookup_query(count, Env, Filter, unordered), + mongoose_rdbms:selected_to_integer(Result). -lookup_query(QueryType, Host, Filters, Order) -> + +lookup_query(QueryType, #{host := Host} = _Env, Filters, Order) -> StmtName = filters_to_statement_name(QueryType, Filters, Order), case mongoose_rdbms:prepared(StmtName) of false -> @@ -511,30 +511,23 @@ lookup_query(QueryType, Host, Filters, Order) -> %% be returned instead. %% @end %% "SELECT COUNT(*) as "index" FROM mam_message WHERE id <= '", UID --spec calc_index(jid:server(), filter(), mod_mam:message_id()) -> non_neg_integer(). -calc_index(Host, Filter, ID) -> - calc_count(Host, to_id(ID, Filter)). +-spec calc_index(env_vars(), filter(), mod_mam:message_id()) -> non_neg_integer(). +calc_index(Env, Filter, ID) -> + calc_count(Env, to_id(ID, Filter)). %% @doc Count of elements in RSet before the passed element. %% %% The element with the passed UID can be already deleted. %% @end %% "SELECT COUNT(*) as "count" FROM mam_message WHERE id < '", UID --spec calc_before(jid:server(), filter(), mod_mam:message_id()) -> non_neg_integer(). -calc_before(Host, Filter, ID) -> - calc_count(Host, before_id(ID, Filter)). +-spec calc_before(env_vars(), filter(), mod_mam:message_id()) -> non_neg_integer(). +calc_before(Env, Filter, ID) -> + calc_count(Env, before_id(ID, Filter)). -%% @doc Get the total result set size. -%% "SELECT COUNT(*) as "count" FROM mam_message WHERE " --spec calc_count(jid:server(), filter()) -> non_neg_integer(). -calc_count(Host, Filter) -> - Result = lookup_query(count, Host, Filter, unordered), - mongoose_rdbms:selected_to_integer(Result). - --spec calc_offset(Host :: jid:server(), +-spec calc_offset(Env :: env_vars(), Filter :: filter(), PageSize :: non_neg_integer(), TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in()) -> non_neg_integer(). -calc_offset(Host, Filter, PageSize, TotalCount, RSM) -> +calc_offset(Env, Filter, PageSize, TotalCount, RSM) -> case RSM of #rsm_in{direction = undefined, index = Index} when is_integer(Index) -> Index; @@ -542,9 +535,9 @@ calc_offset(Host, Filter, PageSize, TotalCount, RSM) -> %% Requesting the Last Page in a Result Set max(0, TotalCount - PageSize); #rsm_in{direction = before, id = ID} when is_integer(ID) -> - max(0, calc_before(Host, Filter, ID) - PageSize); + max(0, calc_before(Env, Filter, ID) - PageSize); #rsm_in{direction = aft, id = ID} when is_integer(ID) -> - calc_index(Host, Filter, ID); + calc_index(Env, Filter, ID); _ -> 0 end. @@ -644,6 +637,15 @@ limit_sql(Filters) -> false -> "" end. +-spec index_hint_sql(jid:server()) -> string(). +index_hint_sql(Host) -> + case mongoose_rdbms:db_engine(Host) of + mysql -> + "USE INDEX(PRIMARY, i_mam_message_rem) "; + _ -> + "" + end. + filters_to_columns(Filters) -> [Column || {_Op, Column, _Value} <- Filters]. From 7f53f15533b05a268483217cc706ca5cf99cf131 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 11:02:54 +0100 Subject: [PATCH 22/92] Put RSM logic into mam_lookup --- src/mam/mam_lookup.erl | 229 +++++++++++++++++++++++++++++++++ src/mam/mod_mam_rdbms_arch.erl | 223 ++------------------------------ 2 files changed, 243 insertions(+), 209 deletions(-) create mode 100644 src/mam/mam_lookup.erl diff --git a/src/mam/mam_lookup.erl b/src/mam/mam_lookup.erl new file mode 100644 index 0000000000..6e25ef1270 --- /dev/null +++ b/src/mam/mam_lookup.erl @@ -0,0 +1,229 @@ +%% RSM logic lives here +-module(mam_lookup). +-export([lookup/3]). + +-type filter() :: list(tuple()). +-type env_vars() :: map(). + +-include("mongoose.hrl"). +-include("jlib.hrl"). +-include_lib("exml/include/exml.hrl"). +-include("mongoose_rsm.hrl"). + +%% Public logic +%% We use two fields from Env: +%% - lookup_fun +%% - row_to_uniform_format_fun +lookup(Env = #{}, Filter, Params = #{rsm := RSM}) when is_list(Filter) -> + OptParams = Params#{opt_count_type => opt_count_type(RSM)}, + choose_lookup_messages_strategy(Env, Filter, OptParams). + +lookup_query(QueryType, #{lookup_fun := LookupF} = Env, Filters, Order) -> + LookupF(QueryType, Env, Filters, Order). + +row_to_uniform_format(Row, #{row_to_uniform_format_fun := FormatF} = Env) -> + FormatF(Row, Env). + +%% Private logic below + +%% Not supported: +%% - #rsm_in{direction = aft, id = ID} +%% - #rsm_in{direction = before, id = ID} +opt_count_type(#rsm_in{direction = before, id = undefined}) -> + last_page; %% last page is supported +opt_count_type(#rsm_in{direction = undefined}) -> + by_offset; %% offset +opt_count_type(undefined) -> + by_offset; %% no RSM +opt_count_type(_) -> + none. %% id field is defined in RSM + +%% There are several strategies how to extract messages: +%% - we can use regular query that requires counting; +%% - we can reduce number of queries if we skip counting for small data sets; +%% - sometimes we want not to count at all +%% (for example, our client side counts ones and keep the information) +choose_lookup_messages_strategy(Env, Filter, + Params = #{rsm := RSM, page_size := PageSize}) -> + case Params of + #{is_simple := true} -> + %% Simple query without calculating offset and total count + simple_lookup_messages(Env, RSM, PageSize, Filter); + %% NOTICE: We always prefer opt_count optimization, if possible. + %% Clients don't event know what opt_count is. + #{opt_count_type := last_page} when PageSize > 0 -> + %% Extract messages before calculating offset and total count + %% Useful for small result sets + lookup_last_page(Env, PageSize, Filter); + #{opt_count_type := by_offset} when PageSize > 0 -> + %% Extract messages before calculating offset and total count + %% Useful for small result sets + lookup_by_offset(Env, RSM, PageSize, Filter); + _ -> + %% Calculate offset and total count first before extracting messages + lookup_messages_regular(Env, RSM, PageSize, Filter) + end. + + +%% Just extract messages +simple_lookup_messages(Env, RSM, PageSize, Filter) -> + {Filter2, Offset, Order} = rsm_to_filter(RSM, Filter), + Messages = extract_messages(Env, Filter2, Offset, PageSize, Order), + {ok, {undefined, undefined, Messages}}. + +rsm_to_filter(RSM, Filter) -> + case RSM of + %% Get last rows from result set + #rsm_in{direction = aft, id = ID} -> + {after_id(ID, Filter), 0, asc}; + #rsm_in{direction = before, id = undefined} -> + {Filter, 0, desc}; + #rsm_in{direction = before, id = ID} -> + {before_id(ID, Filter), 0, desc}; + #rsm_in{direction = undefined, index = Index} -> + {Filter, Index, asc}; + undefined -> + {Filter, 0, asc} + end. + +%% This function handles case: #rsm_in{direction = before, id = undefined} +%% Assumes assert_rsm_without_id(RSM) +lookup_last_page(Env, PageSize, Filter) -> + Messages = extract_messages(Env, Filter, 0, PageSize, desc), + Selected = length(Messages), + Offset = + case Selected < PageSize of + true -> + 0; %% Result fits on a single page + false -> + FirstID = uniform_to_message_id(hd(Messages)), + calc_count(Env, before_id(FirstID, Filter)) + end, + {ok, {Offset + Selected, Offset, Messages}}. + + +lookup_by_offset(Env, RSM, PageSize, Filter) -> + assert_rsm_without_id(RSM), + Offset = rsm_to_index(RSM), + Messages = extract_messages(Env, Filter, Offset, PageSize, asc), + Selected = length(Messages), + TotalCount = + case Selected < PageSize of + true -> + Offset + Selected; %% Result fits on a single page + false -> + LastID = uniform_to_message_id(lists:last(Messages)), + CountAfterLastID = calc_count(Env, after_id(LastID, Filter)), + Offset + Selected + CountAfterLastID + end, + {ok, {TotalCount, Offset, Messages}}. + +assert_rsm_without_id(undefined) -> ok; +assert_rsm_without_id(#rsm_in{id = undefined}) -> ok. + +rsm_to_index(#rsm_in{direction = undefined, index = Offset}) + when is_integer(Offset) -> Offset; +rsm_to_index(_) -> 0. + + +lookup_messages_regular(Env, RSM, PageSize, Filter) -> + TotalCount = calc_count(Env, Filter), + Offset = calc_offset(Env, Filter, PageSize, TotalCount, RSM), + Messages = + case RSM of + #rsm_in{direction = aft, id = ID} -> + extract_messages(Env, from_id(ID, Filter), 0, PageSize + 1, asc); + #rsm_in{direction = before, id = ID} when ID =/= undefined -> + extract_messages(Env, to_id(ID, Filter), 0, PageSize + 1, desc); + _ -> + extract_messages(Env, Filter, Offset, PageSize, asc) + end, + Result = {TotalCount, Offset, Messages}, + mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result). + +-spec after_id(mod_mam:message_id(), filter()) -> filter(). +after_id(ID, Filter) -> + [{greater, id, ID}|Filter]. + +-spec before_id(mod_mam:message_id(), filter()) -> filter(). +before_id(ID, Filter) -> + [{lower, id, ID}|Filter]. + +-spec from_id(mod_mam:message_id(), filter()) -> filter(). +from_id(ID, Filter) -> + [{ge, id, ID}|Filter]. + +-spec to_id(mod_mam:message_id(), filter()) -> filter(). +to_id(ID, Filter) -> + [{le, id, ID}|Filter]. + +rows_to_uniform_format(MessageRows, Env) -> + [row_to_uniform_format(Row, Env) || Row <- MessageRows]. + +uniform_to_message_id({MessID, _, _}) -> MessID. + +-spec extract_messages(Env :: env_vars(), + Filter :: filter(), Offset :: non_neg_integer(), Max :: pos_integer(), + Order :: asc | desc) -> [mod_mam:message_row()]. +extract_messages(_Env, _Filter, _Offset, 0 = _Max, _Order) -> + []; +extract_messages(Env, Filter, Offset, Max, Order) -> + {selected, MessageRows} = extract_rows(Env, Filter, Offset, Max, Order), + ?LOG_DEBUG(#{what => mam_extract_messages, + mam_filter => Filter, offset => Offset, max => Max, + env_vars => Env, message_rows => MessageRows}), + Rows = maybe_reverse(Order, MessageRows), + rows_to_uniform_format(Rows, Env). + +extract_rows(Env, Filters, Offset, Max, Order) -> + Filters2 = Filters ++ rdbms_queries:limit_offset_filters(Max, Offset), + lookup_query(lookup, Env, Filters2, Order). + +%% @doc Get the total result set size. +%% "SELECT COUNT(*) as "count" FROM mam_message WHERE " +-spec calc_count(env_vars(), filter()) -> non_neg_integer(). +calc_count(Env, Filter) -> + Result = lookup_query(count, Env, Filter, unordered), + mongoose_rdbms:selected_to_integer(Result). + +%% @doc Calculate a zero-based index of the row with UID in the result test. +%% +%% If the element does not exists, the ID of the next element will +%% be returned instead. +%% @end +%% "SELECT COUNT(*) as "index" FROM mam_message WHERE id <= '", UID +-spec calc_index(env_vars(), filter(), mod_mam:message_id()) -> non_neg_integer(). +calc_index(Env, Filter, ID) -> + calc_count(Env, to_id(ID, Filter)). + +%% @doc Count of elements in RSet before the passed element. +%% +%% The element with the passed UID can be already deleted. +%% @end +%% "SELECT COUNT(*) as "count" FROM mam_message WHERE id < '", UID +-spec calc_before(env_vars(), filter(), mod_mam:message_id()) -> non_neg_integer(). +calc_before(Env, Filter, ID) -> + calc_count(Env, before_id(ID, Filter)). + +-spec calc_offset(Env :: env_vars(), + Filter :: filter(), PageSize :: non_neg_integer(), + TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in()) -> non_neg_integer(). +calc_offset(Env, Filter, PageSize, TotalCount, RSM) -> + case RSM of + #rsm_in{direction = undefined, index = Index} when is_integer(Index) -> + Index; + #rsm_in{direction = before, id = undefined} -> + %% Requesting the Last Page in a Result Set + max(0, TotalCount - PageSize); + #rsm_in{direction = before, id = ID} when is_integer(ID) -> + max(0, calc_before(Env, Filter, ID) - PageSize); + #rsm_in{direction = aft, id = ID} when is_integer(ID) -> + calc_index(Env, Filter, ID); + _ -> + 0 + end. + +maybe_reverse(asc, List) -> + List; +maybe_reverse(desc, List) -> + lists:reverse(List). diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index dd6344fddf..924f4193b1 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -104,7 +104,7 @@ get_mam_pm_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> undefined -> []; ArchiveID -> Env = env_vars(Host, ArcJID), - {selected, Rows} = extract_gdpr_messages(Host, ArchiveID), + {selected, Rows} = extract_gdpr_messages(Env, ArchiveID), Messages = rows_to_uniform_format(Rows, Env), [uniform_to_gdpr(M) || M <- Messages] ++ Acc end. @@ -153,10 +153,11 @@ encode_direction(outgoing) -> <<"O">>. -spec archive_size(Size :: integer(), Host :: jid:server(), ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). -archive_size(Size, Host, UserID, _ArcJID) when is_integer(Size) -> +archive_size(Size, Host, UserID, ArcJID) when is_integer(Size) -> Filter = [{equal, user_id, UserID}], - Env = #{host => Host}, - calc_count(Env, Filter). + Env = env_vars(Host, ArcJID), + Result = lookup_query(count, Env, Filter, unordered), + mongoose_rdbms:selected_to_integer(Result). -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok. @@ -290,9 +291,8 @@ remove_archive(Host, UserID) -> {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_archive_remove, [UserID]). %% GDPR logic -extract_gdpr_messages(Host, ArchiveID) -> +extract_gdpr_messages(Env, ArchiveID) -> Filters = [{equal, user_id, ArchiveID}], - Env = #{host => Host}, lookup_query(lookup, Env, Filters, asc). %% Lookup logic @@ -305,139 +305,11 @@ lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), ExtParams = extend_params(Params, Env), Filter = prepare_filter(ExtParams), - choose_lookup_messages_strategy(Env, Filter, ExtParams) + mam_lookup:lookup(Env, Filter, ExtParams) catch _Type:Reason:S -> {error, {Reason, {stacktrace, S}}} end. -%% Not supported: -%% - #rsm_in{direction = aft, id = ID} -%% - #rsm_in{direction = before, id = ID} -opt_count_type(#rsm_in{direction = before, id = undefined}) -> - last_page; %% last page is supported -opt_count_type(#rsm_in{direction = undefined}) -> - by_offset; %% offset -opt_count_type(undefined) -> - by_offset; %% no RSM -opt_count_type(_) -> - none. %% id field is defined in RSM - -%% There are several strategies how to extract messages: -%% - we can use regular query that requires counting; -%% - we can reduce number of queries if we skip counting for small data sets; -%% - sometimes we want not to count at all -%% (for example, our client side counts ones and keep the information) -choose_lookup_messages_strategy(Env, Filter, - Params = #{rsm := RSM, page_size := PageSize}) -> - case Params of - #{is_simple := true} -> - %% Simple query without calculating offset and total count - simple_lookup_messages(Env, RSM, PageSize, Filter); - %% NOTICE: We always prefer opt_count optimization, if possible. - %% Clients don't event know what opt_count is. - #{opt_count_type := last_page} when PageSize > 0 -> - %% Extract messages before calculating offset and total count - %% Useful for small result sets - lookup_last_page(Env, PageSize, Filter); - #{opt_count_type := by_offset} when PageSize > 0 -> - %% Extract messages before calculating offset and total count - %% Useful for small result sets - lookup_by_offset(Env, RSM, PageSize, Filter); - _ -> - %% Calculate offset and total count first before extracting messages - lookup_messages_regular(Env, RSM, PageSize, Filter) - end. - -%% Just extract messages -simple_lookup_messages(Env, RSM, PageSize, Filter) -> - {Filter2, Offset, Order} = rsm_to_filter(RSM, Filter), - Messages = extract_messages(Env, Filter2, Offset, PageSize, Order), - {ok, {undefined, undefined, Messages}}. - -rsm_to_filter(RSM, Filter) -> - case RSM of - %% Get last rows from result set - #rsm_in{direction = aft, id = ID} -> - {after_id(ID, Filter), 0, asc}; - #rsm_in{direction = before, id = undefined} -> - {Filter, 0, desc}; - #rsm_in{direction = before, id = ID} -> - {before_id(ID, Filter), 0, desc}; - #rsm_in{direction = undefined, index = Index} -> - {Filter, Index, asc}; - undefined -> - {Filter, 0, asc} - end. - -%% This function handles case: #rsm_in{direction = before, id = undefined} -%% Assumes assert_rsm_without_id(RSM) -lookup_last_page(Env, PageSize, Filter) -> - Messages = extract_messages(Env, Filter, 0, PageSize, desc), - Selected = length(Messages), - Offset = - case Selected < PageSize of - true -> - 0; %% Result fits on a single page - false -> - FirstID = uniform_to_message_id(hd(Messages)), - calc_count(Env, before_id(FirstID, Filter)) - end, - {ok, {Offset + Selected, Offset, Messages}}. - -lookup_by_offset(Env, RSM, PageSize, Filter) -> - assert_rsm_without_id(RSM), - Offset = rsm_to_index(RSM), - Messages = extract_messages(Env, Filter, Offset, PageSize, asc), - Selected = length(Messages), - TotalCount = - case Selected < PageSize of - true -> - Offset + Selected; %% Result fits on a single page - false -> - LastID = uniform_to_message_id(lists:last(Messages)), - CountAfterLastID = calc_count(Env, after_id(LastID, Filter)), - Offset + Selected + CountAfterLastID - end, - {ok, {TotalCount, Offset, Messages}}. - -assert_rsm_without_id(undefined) -> ok; -assert_rsm_without_id(#rsm_in{id = undefined}) -> ok. - -rsm_to_index(#rsm_in{direction = undefined, index = Offset}) - when is_integer(Offset) -> Offset; -rsm_to_index(_) -> 0. - -lookup_messages_regular(Env, RSM, PageSize, Filter) -> - TotalCount = calc_count(Env, Filter), - Offset = calc_offset(Env, Filter, PageSize, TotalCount, RSM), - Messages = - case RSM of - #rsm_in{direction = aft, id = ID} -> - extract_messages(Env, from_id(ID, Filter), 0, PageSize + 1, asc); - #rsm_in{direction = before, id = ID} when ID =/= undefined -> - extract_messages(Env, to_id(ID, Filter), 0, PageSize + 1, desc); - _ -> - extract_messages(Env, Filter, Offset, PageSize, asc) - end, - Result = {TotalCount, Offset, Messages}, - mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result). - --spec after_id(mod_mam:message_id(), filter()) -> filter(). -after_id(ID, Filter) -> - [{greater, id, ID}|Filter]. - --spec before_id(mod_mam:message_id(), filter()) -> filter(). -before_id(ID, Filter) -> - [{lower, id, ID}|Filter]. - --spec from_id(mod_mam:message_id(), filter()) -> filter(). -from_id(ID, Filter) -> - [{ge, id, ID}|Filter]. - --spec to_id(mod_mam:message_id(), filter()) -> filter(). -to_id(ID, Filter) -> - [{le, id, ID}|Filter]. - rows_to_uniform_format(MessageRows, Env) -> [row_to_uniform_format(Row, Env) || Row <- MessageRows]. @@ -448,33 +320,6 @@ row_to_uniform_format({BMessID, BSrcJID, SDataRaw}, Env) -> Packet = decode_packet(Data, Env), {MessID, SrcJID, Packet}. -uniform_to_message_id({MessID, _, _}) -> MessID. - --spec extract_messages(Env :: env_vars(), - Filter :: filter(), Offset :: non_neg_integer(), Max :: pos_integer(), - Order :: asc | desc) -> [mod_mam:message_row()]. -extract_messages(_Env, _Filter, _Offset, 0 = _Max, _Order) -> - []; -extract_messages(Env, Filter, Offset, Max, Order) -> - {selected, MessageRows} = extract_rows(Env, Filter, Offset, Max, Order), - ?LOG_DEBUG(#{what => mam_extract_messages, - mam_filter => Filter, offset => Offset, max => Max, - env_vars => Env, message_rows => MessageRows}), - Rows = maybe_reverse(Order, MessageRows), - rows_to_uniform_format(Rows, Env). - -extract_rows(Env, Filters, Offset, Max, Order) -> - Filters2 = Filters ++ rdbms_queries:limit_offset_filters(Max, Offset), - lookup_query(lookup, Env, Filters2, Order). - -%% @doc Get the total result set size. -%% "SELECT COUNT(*) as "count" FROM mam_message WHERE " --spec calc_count(env_vars(), filter()) -> non_neg_integer(). -calc_count(Env, Filter) -> - Result = lookup_query(count, Env, Filter, unordered), - mongoose_rdbms:selected_to_integer(Result). - - lookup_query(QueryType, #{host := Host} = _Env, Filters, Order) -> StmtName = filters_to_statement_name(QueryType, Filters, Order), case mongoose_rdbms:prepared(StmtName) of @@ -505,47 +350,6 @@ lookup_query(QueryType, #{host := Host} = _Env, Filters, Order) -> error(What) end. -%% @doc Calculate a zero-based index of the row with UID in the result test. -%% -%% If the element does not exists, the ID of the next element will -%% be returned instead. -%% @end -%% "SELECT COUNT(*) as "index" FROM mam_message WHERE id <= '", UID --spec calc_index(env_vars(), filter(), mod_mam:message_id()) -> non_neg_integer(). -calc_index(Env, Filter, ID) -> - calc_count(Env, to_id(ID, Filter)). - -%% @doc Count of elements in RSet before the passed element. -%% -%% The element with the passed UID can be already deleted. -%% @end -%% "SELECT COUNT(*) as "count" FROM mam_message WHERE id < '", UID --spec calc_before(env_vars(), filter(), mod_mam:message_id()) -> non_neg_integer(). -calc_before(Env, Filter, ID) -> - calc_count(Env, before_id(ID, Filter)). - --spec calc_offset(Env :: env_vars(), - Filter :: filter(), PageSize :: non_neg_integer(), - TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in()) -> non_neg_integer(). -calc_offset(Env, Filter, PageSize, TotalCount, RSM) -> - case RSM of - #rsm_in{direction = undefined, index = Index} when is_integer(Index) -> - Index; - #rsm_in{direction = before, id = undefined} -> - %% Requesting the Last Page in a Result Set - max(0, TotalCount - PageSize); - #rsm_in{direction = before, id = ID} when is_integer(ID) -> - max(0, calc_before(Env, Filter, ID) - PageSize); - #rsm_in{direction = aft, id = ID} when is_integer(ID) -> - calc_index(Env, Filter, ID); - _ -> - 0 - end. - -maybe_reverse(asc, List) -> - List; -maybe_reverse(desc, List) -> - lists:reverse(List). %% ---------------------------------------------------------------------- %% Optimizations and extensible code @@ -563,9 +367,9 @@ maybe_encode_compact_uuid(undefined, _) -> maybe_encode_compact_uuid(Microseconds, NodeID) -> encode_compact_uuid(Microseconds, NodeID). -maybe_minify_bare_jid(undefined, _Env) -> +maybe_encode_bare_jid(undefined, _Env) -> undefined; -maybe_minify_bare_jid(JID, Env) -> +maybe_encode_bare_jid(JID, Env) -> encode_jid(jid:to_bare(JID), Env). -spec encode_jid(jid:jid(), env_vars()) -> binary(). @@ -598,6 +402,8 @@ env_vars(Host, ArcJID) -> %% It's only for passing into RDBMS. #{host => Host, archive_jid => ArcJID, + lookup_fun => fun lookup_query/4, + row_to_uniform_format_fun => fun row_to_uniform_format/2, has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam, Host), full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), db_jid_codec => db_jid_codec(Host, ?MODULE), @@ -719,14 +525,13 @@ filter_to_sql(equal, Column) -> filter_to_sql(like, Column) -> Column ++ " LIKE ?". -extend_params(#{rsm := RSM, borders := Borders, +extend_params(#{borders := Borders, start_ts := Start, end_ts := End, with_jid := WithJID, search_text := SearchText} = Params, Env) -> - Params#{opt_count_type => opt_count_type(RSM), - norm_search_text => mod_mam_utils:normalize_search_text(SearchText), + Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText), start_id => make_start_id(Start, Borders), end_id => make_end_id(End, Borders), - remote_bare_jid => maybe_minify_bare_jid(WithJID, Env), + remote_bare_jid => maybe_encode_bare_jid(WithJID, Env), remote_resource => jid_to_non_empty_resource(WithJID)}. jid_to_non_empty_resource(#jid{lresource = Res}) when byte_size(Res) > 0 -> From 4473a214e127efaebe2d3e40e0721ec423abd0df Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 12:44:12 +0100 Subject: [PATCH 23/92] Prettify mam_lookup --- src/mam/mam_lookup.erl | 100 ++++++++++++++++++--------------- src/mam/mod_mam_rdbms_arch.erl | 26 ++++----- src/mam/mod_mam_utils.erl | 4 -- 3 files changed, 64 insertions(+), 66 deletions(-) diff --git a/src/mam/mam_lookup.erl b/src/mam/mam_lookup.erl index 6e25ef1270..18fa6c00fd 100644 --- a/src/mam/mam_lookup.erl +++ b/src/mam/mam_lookup.erl @@ -2,18 +2,23 @@ -module(mam_lookup). -export([lookup/3]). --type filter() :: list(tuple()). --type env_vars() :: map(). - -include("mongoose.hrl"). -include("jlib.hrl"). --include_lib("exml/include/exml.hrl"). -include("mongoose_rsm.hrl"). +-type filter() :: list(tuple()). +-type env_vars() :: map(). +-type params() :: map(). +-type message_id() :: mod_mam:message_id(). +-type maybe_rsm() :: #rsm_in{} | undefined. +-type opt_count_type() :: last_page | by_offset | none. + %% Public logic %% We use two fields from Env: %% - lookup_fun %% - row_to_uniform_format_fun +-spec lookup(env_vars(), filter(), params()) -> + {ok, mod_mam:lookup_result()} | {error, item_not_found}. lookup(Env = #{}, Filter, Params = #{rsm := RSM}) when is_list(Filter) -> OptParams = Params#{opt_count_type => opt_count_type(RSM)}, choose_lookup_messages_strategy(Env, Filter, OptParams). @@ -26,9 +31,10 @@ row_to_uniform_format(Row, #{row_to_uniform_format_fun := FormatF} = Env) -> %% Private logic below -%% Not supported: +%% There is no optimizations for these queries yet: %% - #rsm_in{direction = aft, id = ID} %% - #rsm_in{direction = before, id = ID} +-spec opt_count_type(RSM :: maybe_rsm()) -> opt_count_type(). opt_count_type(#rsm_in{direction = before, id = undefined}) -> last_page; %% last page is supported opt_count_type(#rsm_in{direction = undefined}) -> @@ -64,8 +70,7 @@ choose_lookup_messages_strategy(Env, Filter, lookup_messages_regular(Env, RSM, PageSize, Filter) end. - -%% Just extract messages +%% Just extract messages without count and offset information simple_lookup_messages(Env, RSM, PageSize, Filter) -> {Filter2, Offset, Order} = rsm_to_filter(RSM, Filter), Messages = extract_messages(Env, Filter2, Offset, PageSize, Order), @@ -101,7 +106,6 @@ lookup_last_page(Env, PageSize, Filter) -> end, {ok, {Offset + Selected, Offset, Messages}}. - lookup_by_offset(Env, RSM, PageSize, Filter) -> assert_rsm_without_id(RSM), Offset = rsm_to_index(RSM), @@ -125,37 +129,30 @@ rsm_to_index(#rsm_in{direction = undefined, index = Offset}) when is_integer(Offset) -> Offset; rsm_to_index(_) -> 0. - lookup_messages_regular(Env, RSM, PageSize, Filter) -> TotalCount = calc_count(Env, Filter), Offset = calc_offset(Env, Filter, PageSize, TotalCount, RSM), - Messages = - case RSM of - #rsm_in{direction = aft, id = ID} -> - extract_messages(Env, from_id(ID, Filter), 0, PageSize + 1, asc); - #rsm_in{direction = before, id = ID} when ID =/= undefined -> - extract_messages(Env, to_id(ID, Filter), 0, PageSize + 1, desc); - _ -> - extract_messages(Env, Filter, Offset, PageSize, asc) - end, + {LookupById, Filter2, Offset2, PageSize2, Order} = + rsm_to_regular_lookup_vars(RSM, Filter, Offset, PageSize), + Messages = extract_messages(Env, Filter2, Offset2, PageSize2, Order), Result = {TotalCount, Offset, Messages}, - mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result). - --spec after_id(mod_mam:message_id(), filter()) -> filter(). -after_id(ID, Filter) -> - [{greater, id, ID}|Filter]. - --spec before_id(mod_mam:message_id(), filter()) -> filter(). -before_id(ID, Filter) -> - [{lower, id, ID}|Filter]. - --spec from_id(mod_mam:message_id(), filter()) -> filter(). -from_id(ID, Filter) -> - [{ge, id, ID}|Filter]. + case LookupById of + true -> %% check if we've selected a message with #rsm_in.id + mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); + false -> + {ok, Result} + end. --spec to_id(mod_mam:message_id(), filter()) -> filter(). -to_id(ID, Filter) -> - [{le, id, ID}|Filter]. +rsm_to_regular_lookup_vars(RSM, Filter, Offset, PageSize) -> + case RSM of + #rsm_in{direction = aft, id = ID} when ID =/= undefined -> + %% Set extra flag when selecting PageSize + 1 messages + {true, from_id(ID, Filter), 0, PageSize + 1, asc}; + #rsm_in{direction = before, id = ID} when ID =/= undefined -> + {true, to_id(ID, Filter), 0, PageSize + 1, desc}; + _ -> + {false, Filter, Offset, PageSize, asc} + end. rows_to_uniform_format(MessageRows, Env) -> [row_to_uniform_format(Row, Env) || Row <- MessageRows]. @@ -169,18 +166,18 @@ extract_messages(_Env, _Filter, _Offset, 0 = _Max, _Order) -> []; extract_messages(Env, Filter, Offset, Max, Order) -> {selected, MessageRows} = extract_rows(Env, Filter, Offset, Max, Order), - ?LOG_DEBUG(#{what => mam_extract_messages, - mam_filter => Filter, offset => Offset, max => Max, - env_vars => Env, message_rows => MessageRows}), Rows = maybe_reverse(Order, MessageRows), rows_to_uniform_format(Rows, Env). +maybe_reverse(asc, List) -> List; +maybe_reverse(desc, List) -> lists:reverse(List). + extract_rows(Env, Filters, Offset, Max, Order) -> Filters2 = Filters ++ rdbms_queries:limit_offset_filters(Max, Offset), lookup_query(lookup, Env, Filters2, Order). %% @doc Get the total result set size. -%% "SELECT COUNT(*) as "count" FROM mam_message WHERE " +%% SELECT COUNT(*) as count FROM mam_message -spec calc_count(env_vars(), filter()) -> non_neg_integer(). calc_count(Env, Filter) -> Result = lookup_query(count, Env, Filter, unordered), @@ -191,8 +188,8 @@ calc_count(Env, Filter) -> %% If the element does not exists, the ID of the next element will %% be returned instead. %% @end -%% "SELECT COUNT(*) as "index" FROM mam_message WHERE id <= '", UID --spec calc_index(env_vars(), filter(), mod_mam:message_id()) -> non_neg_integer(). +%% SELECT COUNT(*) as index FROM mam_message WHERE id <= ? +-spec calc_index(env_vars(), filter(), message_id()) -> non_neg_integer(). calc_index(Env, Filter, ID) -> calc_count(Env, to_id(ID, Filter)). @@ -200,8 +197,8 @@ calc_index(Env, Filter, ID) -> %% %% The element with the passed UID can be already deleted. %% @end -%% "SELECT COUNT(*) as "count" FROM mam_message WHERE id < '", UID --spec calc_before(env_vars(), filter(), mod_mam:message_id()) -> non_neg_integer(). +%% SELECT COUNT(*) as count FROM mam_message WHERE id < ? +-spec calc_before(env_vars(), filter(), message_id()) -> non_neg_integer(). calc_before(Env, Filter, ID) -> calc_count(Env, before_id(ID, Filter)). @@ -223,7 +220,18 @@ calc_offset(Env, Filter, PageSize, TotalCount, RSM) -> 0 end. -maybe_reverse(asc, List) -> - List; -maybe_reverse(desc, List) -> - lists:reverse(List). +-spec after_id(message_id(), filter()) -> filter(). +after_id(ID, Filter) -> + [{greater, id, ID}|Filter]. + +-spec before_id(message_id(), filter()) -> filter(). +before_id(ID, Filter) -> + [{lower, id, ID}|Filter]. + +-spec from_id(message_id(), filter()) -> filter(). +from_id(ID, Filter) -> + [{ge, id, ID}|Filter]. + +-spec to_id(message_id(), filter()) -> filter(). +to_id(ID, Filter) -> + [{le, id, ID}|Filter]. diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 924f4193b1..672c137943 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -105,8 +105,7 @@ get_mam_pm_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> ArchiveID -> Env = env_vars(Host, ArcJID), {selected, Rows} = extract_gdpr_messages(Env, ArchiveID), - Messages = rows_to_uniform_format(Rows, Env), - [uniform_to_gdpr(M) || M <- Messages] ++ Acc + [uniform_to_gdpr(row_to_uniform_format(Row, Env)) || Row <- Rows] ++ Acc end. uniform_to_gdpr({MessID, RemoteJID, Packet}) -> @@ -147,10 +146,6 @@ stop_pm(Host) -> %% ---------------------------------------------------------------------- %% Internal functions and callbacks -encode_direction(incoming) -> <<"I">>; -encode_direction(outgoing) -> <<"O">>. - - -spec archive_size(Size :: integer(), Host :: jid:server(), ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). archive_size(Size, Host, UserID, ArcJID) when is_integer(Size) -> @@ -159,12 +154,12 @@ archive_size(Size, Host, UserID, ArcJID) when is_integer(Size) -> Result = lookup_query(count, Env, Filter, unordered), mongoose_rdbms:selected_to_integer(Result). - -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok. archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> try Env = env_vars(Host, ArcJID), - do_archive_message(Host, Params, Env) + do_archive_message(Host, Params, Env), + retract_message(Host, Params, Env) catch Class:Reason:StackTrace -> ?LOG_ERROR(#{what => archive_message_failed, host => Host, mam_params => Params, @@ -174,10 +169,10 @@ archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> do_archive_message(Host, Params, Env) -> Row = prepare_message_with_env(Params, Env), - {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_message, Row), - retract_message(Host, Params, Env). + {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_message, Row). %% Retraction logic +%% Called after inserting a new message -spec retract_message(jid:server(), mod_mam:archive_message_params()) -> ok. retract_message(Host, #{local_jid := ArcJID} = Params) -> Env = env_vars(Host, ArcJID), @@ -310,9 +305,6 @@ lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> {error, {Reason, {stacktrace, S}}} end. -rows_to_uniform_format(MessageRows, Env) -> - [row_to_uniform_format(Row, Env) || Row <- MessageRows]. - row_to_uniform_format({BMessID, BSrcJID, SDataRaw}, Env) -> MessID = mongoose_rdbms:result_to_integer(BMessID), SrcJID = decode_jid(BSrcJID, Env), @@ -354,6 +346,9 @@ lookup_query(QueryType, #{host := Host} = _Env, Filters, Order) -> %% ---------------------------------------------------------------------- %% Optimizations and extensible code +encode_direction(incoming) -> <<"I">>; +encode_direction(outgoing) -> <<"O">>. + make_start_id(Start, Borders) -> StartID = maybe_encode_compact_uuid(Start, 0), apply_start_border(Borders, StartID). @@ -525,9 +520,8 @@ filter_to_sql(equal, Column) -> filter_to_sql(like, Column) -> Column ++ " LIKE ?". -extend_params(#{borders := Borders, - start_ts := Start, end_ts := End, with_jid := WithJID, - search_text := SearchText} = Params, Env) -> +extend_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, + borders := Borders, search_text := SearchText} = Params, Env) -> Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText), start_id => make_start_id(Start, Borders), end_id => make_end_id(End, Borders), diff --git a/src/mam/mod_mam_utils.erl b/src/mam/mod_mam_utils.erl index b30eba9582..ef38fa3184 100644 --- a/src/mam/mod_mam_utils.erl +++ b/src/mam/mod_mam_utils.erl @@ -1156,10 +1156,6 @@ is_policy_violation(TotalCount, Offset, MaxResultLimit, LimitPassed) -> PageSize :: non_neg_integer(), LookupResult :: mod_mam:lookup_result(), R :: {ok, mod_mam:lookup_result()} | {error, item_not_found}. -check_for_item_not_found(undefined, _PageSize, Result) -> - {ok, Result}; -check_for_item_not_found(#rsm_in{id = undefined}, _PageSize, Result) -> - {ok, Result}; check_for_item_not_found(#rsm_in{direction = before, id = ID}, PageSize, {TotalCount, Offset, MessageRows}) -> case maybe_last(MessageRows) of From 3de2dad232402e49e40c011f732bb84b8a52ea5b Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 12:59:16 +0100 Subject: [PATCH 24/92] Put code on one line where is possible in mod_mam_rdbms_arch --- src/mam/mod_mam_rdbms_arch.erl | 63 ++++++++++++---------------------- 1 file changed, 21 insertions(+), 42 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 672c137943..0be8d2e2d8 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -362,10 +362,8 @@ maybe_encode_compact_uuid(undefined, _) -> maybe_encode_compact_uuid(Microseconds, NodeID) -> encode_compact_uuid(Microseconds, NodeID). -maybe_encode_bare_jid(undefined, _Env) -> - undefined; -maybe_encode_bare_jid(JID, Env) -> - encode_jid(jid:to_bare(JID), Env). +maybe_encode_bare_jid(undefined, _Env) -> undefined; +maybe_encode_bare_jid(JID, Env) -> encode_jid(jid:to_bare(JID), Env). -spec encode_jid(jid:jid(), env_vars()) -> binary(). encode_jid(JID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> @@ -422,8 +420,7 @@ lookup_sql(QueryType, Filters, Order, IndexHintSQL) -> LimitSQL = limit_sql(Filters), OrderSQL = order_to_sql(Order), FilterSQL = filters_to_sql(Filters), - ["SELECT ", columns_sql(QueryType), " " - "FROM mam_message ", + ["SELECT ", columns_sql(QueryType), " FROM mam_message ", IndexHintSQL, FilterSQL, OrderSQL, LimitSQL]. columns_sql(lookup) -> "id, from_jid, message"; @@ -441,10 +438,8 @@ limit_sql(Filters) -> -spec index_hint_sql(jid:server()) -> string(). index_hint_sql(Host) -> case mongoose_rdbms:db_engine(Host) of - mysql -> - "USE INDEX(PRIMARY, i_mam_message_rem) "; - _ -> - "" + mysql -> "USE INDEX(PRIMARY, i_mam_message_rem) "; + _ -> "" end. filters_to_columns(Filters) -> @@ -491,34 +486,22 @@ column_to_id(offset) -> "o". filters_to_sql(Filters) -> SQLs = [filter_to_sql(Filter) || Filter <- Filters], case skip_undefined(SQLs) of - [] -> - ""; - Defined -> - [" WHERE ", rdbms_queries:join(Defined, " AND ")] + [] -> ""; + Defined -> [" WHERE ", rdbms_queries:join(Defined, " AND ")] end. -skip_undefined(List) -> - [X || X <- List, X =/= undefined]. +skip_undefined(List) -> [X || X <- List, X =/= undefined]. -filter_to_sql({Op, Column, _Value}) -> - filter_to_sql(Op, atom_to_list(Column)). +filter_to_sql({Op, Column, _Value}) -> filter_to_sql(atom_to_list(Column), Op). -filter_to_sql(limit, _) -> - undefined; -filter_to_sql(offset, _) -> - undefined; -filter_to_sql(lower, Column) -> - Column ++ " < ?"; -filter_to_sql(greater, Column) -> - Column ++ " > ?"; -filter_to_sql(le, Column) -> - Column ++ " <= ?"; -filter_to_sql(ge, Column) -> - Column ++ " >= ?"; -filter_to_sql(equal, Column) -> - Column ++ " = ?"; -filter_to_sql(like, Column) -> - Column ++ " LIKE ?". +filter_to_sql(_Column, limit) -> undefined; +filter_to_sql(_Column, offset) -> undefined; +filter_to_sql(Column, lower) -> Column ++ " < ?"; +filter_to_sql(Column, greater) -> Column ++ " > ?"; +filter_to_sql(Column, le) -> Column ++ " <= ?"; +filter_to_sql(Column, ge) -> Column ++ " >= ?"; +filter_to_sql(Column, equal) -> Column ++ " = ?"; +filter_to_sql(Column, like) -> Column ++ " LIKE ?". extend_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, borders := Borders, search_text := SearchText} = Params, Env) -> @@ -528,10 +511,8 @@ extend_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, remote_bare_jid => maybe_encode_bare_jid(WithJID, Env), remote_resource => jid_to_non_empty_resource(WithJID)}. -jid_to_non_empty_resource(#jid{lresource = Res}) when byte_size(Res) > 0 -> - Res; -jid_to_non_empty_resource(_) -> - undefined. +jid_to_non_empty_resource(#jid{lresource = Res}) when byte_size(Res) > 0 -> Res; +jid_to_non_empty_resource(_) -> undefined. -record(lookup_field, {op, column, param, required, value_maker}). @@ -558,10 +539,8 @@ field_to_values(#lookup_field{param = Param, value_maker = ValueMaker, required [] end. -make_value(search_words, Value) -> - search_words(Value); -make_value(undefined, Value) -> - [Value]. +make_value(search_words, Value) -> search_words(Value); +make_value(undefined, Value) -> [Value]. new_filter(#lookup_field{op = Op, column = Column}, Value) -> {Op, Column, Value}. From d90474dc383c864cf404706ea68d7a28d37ae6ef Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 13:38:44 +0100 Subject: [PATCH 25/92] Move some code into encode_search_body --- src/mam/mod_mam_rdbms_arch.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 0be8d2e2d8..4d0a287386 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -259,8 +259,8 @@ encode_value(jid_resource, #jid{lresource = Res}, _Env) -> Res; encode_value(xml, Value, Env) -> encode_packet(Value, Env); -encode_value(search, Value, #{full_text_search := SearchEnabled}) -> - mod_mam_utils:packet_to_search_body(SearchEnabled, Value). +encode_value(search, Value, Env) -> + encode_search_body(Value, Env). column_names() -> [Column || #db_mapping{column = Column} <- db_mappings()]. @@ -390,6 +390,10 @@ unescape_binary(Bin, #{host := Host}) -> get_retract_id(Packet, #{has_message_retraction := Enabled}) -> mod_mam_utils:get_retract_id(Enabled, Packet). +-spec encode_search_body(exml:element(), env_vars()) -> binary(). +encode_search_body(Packet, #{full_text_search := SearchEnabled}) -> + mod_mam_utils:packet_to_search_body(SearchEnabled, Packet). + env_vars(Host, ArcJID) -> %% Please, minimize usage of the host field. %% It's only for passing into RDBMS. From 7c5c150abb1ab4e5bc6af6054c3a421675cd932f Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 15:00:07 +0100 Subject: [PATCH 26/92] Reorder some functions --- src/mam/mod_mam_rdbms_arch.erl | 203 ++++++++++++++++----------------- 1 file changed, 101 insertions(+), 102 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 4d0a287386..357e47fd55 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -66,6 +66,50 @@ -type env_vars() :: map(). +-type value_type() :: int | maybe_string | direction | bare_jid | jid | jid_resource | xml | search. + +-record(db_mapping, {column, param, format}). +-record(lookup_field, {op, column, param, required, value_maker}). + +db_mappings() -> + [#db_mapping{column = id, param = message_id, format = int}, + #db_mapping{column = user_id, param = archive_id, format = int}, + #db_mapping{column = remote_bare_jid, param = remote_jid, format = bare_jid}, + #db_mapping{column = remote_resource, param = remote_jid, format = jid_resource}, + #db_mapping{column = direction, param = direction, format = direction}, + #db_mapping{column = from_jid, param = source_jid, format = jid}, + #db_mapping{column = origin_id, param = origin_id, format = maybe_string}, + #db_mapping{column = message, param = packet, format = xml}, + #db_mapping{column = search_body, param = packet, format = search}]. + +lookup_fields() -> + [#lookup_field{op = equal, column = user_id, param = archive_id, required = true}, + #lookup_field{op = ge, column = id, param = start_id}, + #lookup_field{op = le, column = id, param = end_id}, + #lookup_field{op = equal, column = remote_bare_jid, param = remote_bare_jid}, + #lookup_field{op = equal, column = remote_resource, param = remote_resource}, + #lookup_field{op = like, column = search_body, param = norm_search_text, value_maker = search_words}]. + +env_vars(Host, ArcJID) -> + %% Please, minimize the usage of the host field. + %% It's only for passing into RDBMS. + #{host => Host, + archive_jid => ArcJID, + lookup_fun => fun lookup_query/4, + row_to_uniform_format_fun => fun row_to_uniform_format/2, + has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam, Host), + full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), + db_jid_codec => db_jid_codec(Host, ?MODULE), + db_message_codec => db_message_codec(Host, ?MODULE)}. + +extend_lookup_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, + borders := Borders, search_text := SearchText} = Params, Env) -> + Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText), + start_id => make_start_id(Start, Borders), + end_id => make_end_id(End, Borders), + remote_bare_jid => maybe_encode_bare_jid(WithJID, Env), + remote_resource => jid_to_non_empty_resource(WithJID)}. + %% ---------------------------------------------------------------------- %% gen_mod callbacks %% Starting and stopping functions for users' archives @@ -92,7 +136,6 @@ start(Host, Opts) -> " ORDER BY id DESC ", LimitSQL/binary>>]), ok. - -spec stop(jid:server()) -> ok. stop(Host) -> stop_pm(Host). @@ -179,20 +222,18 @@ retract_message(Host, #{local_jid := ArcJID} = Params) -> retract_message(Host, Params, Env). -spec retract_message(jid:server(), mod_mam:archive_message_params(), env_vars()) -> ok. -retract_message(Host, #{archive_id := UserID, - remote_jid := RemJID, - direction := Dir, - packet := Packet}, Env) -> +retract_message(Host, #{archive_id := UserID, remote_jid := RemJID, + direction := Dir, packet := Packet}, Env) -> case get_retract_id(Packet, Env) of none -> ok; - OriginIDToRetract -> retract_message(Host, UserID, RemJID, OriginIDToRetract, Dir, Env) + OriginID -> retract_message(Host, UserID, RemJID, OriginID, Dir, Env) end. retract_message(Host, UserID, RemJID, OriginID, Dir, Env) -> - MinBareRemJID = encode_jid(jid:to_bare(RemJID), Env), + BinBareRemJID = encode_jid(jid:to_bare(RemJID), Env), BinDir = encode_direction(Dir), {selected, Rows} = execute_select_messages_to_retract( - Host, UserID, MinBareRemJID, OriginID, BinDir), + Host, UserID, BinBareRemJID, OriginID, BinDir), make_tombstone(Host, UserID, OriginID, Rows, Env), ok. @@ -218,18 +259,6 @@ execute_make_tombstone(Host, TombstoneData, UserID, MessID) -> [TombstoneData, UserID, MessID]). %% Insert logic --record(db_mapping, {column, param, format}). - -db_mappings() -> - [#db_mapping{column = id, param = message_id, format = int}, - #db_mapping{column = user_id, param = archive_id, format = int}, - #db_mapping{column = remote_bare_jid, param = remote_jid, format = bare_jid}, - #db_mapping{column = remote_resource, param = remote_jid, format = jid_resource}, - #db_mapping{column = direction, param = direction, format = direction}, - #db_mapping{column = from_jid, param = source_jid, format = jid}, - #db_mapping{column = origin_id, param = origin_id, format = maybe_binary}, - #db_mapping{column = message, param = packet, format = xml}, - #db_mapping{column = search_body, param = packet, format = search}]. -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). prepare_message(Host, Params = #{local_jid := ArcJID}) -> @@ -243,32 +272,13 @@ prepare_value(Params, Env, #db_mapping{param = Param, format = Format}) -> Value = maps:get(Param, Params), encode_value(Format, Value, Env). -encode_value(int, Value, _Env) when is_integer(Value) -> - Value; -encode_value(maybe_binary, none, _Env) -> - null; -encode_value(maybe_binary, Value, _Env) when is_binary(Value) -> - Value; -encode_value(direction, Value, _Env) -> - encode_direction(Value); -encode_value(bare_jid, Value, Env) -> - encode_jid(jid:to_bare(Value), Env); -encode_value(jid, Value, Env) -> - encode_jid(Value, Env); -encode_value(jid_resource, #jid{lresource = Res}, _Env) -> - Res; -encode_value(xml, Value, Env) -> - encode_packet(Value, Env); -encode_value(search, Value, Env) -> - encode_search_body(Value, Env). - -column_names() -> - [Column || #db_mapping{column = Column} <- db_mappings()]. +column_names(Mappings) -> + [Column || #db_mapping{column = Column} <- Mappings]. -spec prepare_insert(Name :: atom(), NumRows :: pos_integer()) -> ok. prepare_insert(Name, NumRows) -> Table = mam_message, - Fields = column_names(), + Fields = column_names(db_mappings()), {Query, Fields2} = rdbms_queries:create_bulk_insert_query(Table, Fields, NumRows), mongoose_rdbms:prepare(Name, Table, Fields2, Query), ok. @@ -296,14 +306,10 @@ extract_gdpr_messages(Env, ArchiveID) -> lookup_messages({error, _Reason}=Result, _Host, _Params) -> Result; lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> - try - Env = env_vars(Host, ArcJID), - ExtParams = extend_params(Params, Env), - Filter = prepare_filter(ExtParams), - mam_lookup:lookup(Env, Filter, ExtParams) - catch _Type:Reason:S -> - {error, {Reason, {stacktrace, S}}} - end. + Env = env_vars(Host, ArcJID), + ExtParams = extend_lookup_params(Params, Env), + Filter = prepare_filter(ExtParams), + mam_lookup:lookup(Env, Filter, ExtParams). row_to_uniform_format({BMessID, BSrcJID, SDataRaw}, Env) -> MessID = mongoose_rdbms:result_to_integer(BMessID), @@ -346,6 +352,26 @@ lookup_query(QueryType, #{host := Host} = _Env, Filters, Order) -> %% ---------------------------------------------------------------------- %% Optimizations and extensible code +-spec encode_value(value_type(), term(), env_vars()) -> term(). +encode_value(int, Value, _Env) when is_integer(Value) -> + Value; +encode_value(maybe_string, none, _Env) -> + null; +encode_value(maybe_string, Value, _Env) when is_binary(Value) -> + Value; +encode_value(direction, Value, _Env) -> + encode_direction(Value); +encode_value(bare_jid, Value, Env) -> + encode_jid(jid:to_bare(Value), Env); +encode_value(jid, Value, Env) -> + encode_jid(Value, Env); +encode_value(jid_resource, #jid{lresource = Res}, _Env) -> + Res; +encode_value(xml, Value, Env) -> + encode_packet(Value, Env); +encode_value(search, Value, Env) -> + encode_search_body(Value, Env). + encode_direction(incoming) -> <<"I">>; encode_direction(outgoing) -> <<"O">>. @@ -362,6 +388,10 @@ maybe_encode_compact_uuid(undefined, _) -> maybe_encode_compact_uuid(Microseconds, NodeID) -> encode_compact_uuid(Microseconds, NodeID). +jid_to_non_empty_resource(undefined) -> undefined; +jid_to_non_empty_resource(#jid{lresource = <<>>}) -> undefined; +jid_to_non_empty_resource(#jid{lresource = Res}) -> Res. + maybe_encode_bare_jid(undefined, _Env) -> undefined; maybe_encode_bare_jid(JID, Env) -> encode_jid(jid:to_bare(JID), Env). @@ -394,18 +424,6 @@ get_retract_id(Packet, #{has_message_retraction := Enabled}) -> encode_search_body(Packet, #{full_text_search := SearchEnabled}) -> mod_mam_utils:packet_to_search_body(SearchEnabled, Packet). -env_vars(Host, ArcJID) -> - %% Please, minimize usage of the host field. - %% It's only for passing into RDBMS. - #{host => Host, - archive_jid => ArcJID, - lookup_fun => fun lookup_query/4, - row_to_uniform_format_fun => fun row_to_uniform_format/2, - has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam, Host), - full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), - db_jid_codec => db_jid_codec(Host, ?MODULE), - db_message_codec => db_message_codec(Host, ?MODULE)}. - -spec db_jid_codec(jid:server(), module()) -> module(). db_jid_codec(Host, Module) -> gen_mod:get_module_opt(Host, Module, db_jid_format, mam_jid_mini). @@ -427,14 +445,12 @@ lookup_sql(QueryType, Filters, Order, IndexHintSQL) -> ["SELECT ", columns_sql(QueryType), " FROM mam_message ", IndexHintSQL, FilterSQL, OrderSQL, LimitSQL]. -columns_sql(lookup) -> "id, from_jid, message"; -columns_sql(count) -> "COUNT(*)". - %% Caller should provide both limit and offset fields in the correct order. %% See limit_offset_filters. %% No limits option is fine too (it is used with count and GDPR). limit_sql(Filters) -> - case lists:keymember(limit, 1, Filters) of %% and offset + HasLimit = lists:keymember(limit, 1, Filters), %% and offset + case HasLimit of true -> rdbms_queries:limit_offset_sql(); false -> "" end. @@ -454,10 +470,13 @@ filters_to_args(Filters) -> filters_to_statement_name(QueryType, Filters, Order) -> QueryId = query_type_to_id(QueryType), - Ids = [type_to_id(Type) ++ column_to_id(Col) || {Type, Col, _Val} <- Filters], + Ids = [op_to_id(Op) ++ column_to_id(Col) || {Op, Col, _Val} <- Filters], OrderId = order_type_to_id(Order), list_to_atom("mam_" ++ QueryId ++ "_" ++ OrderId ++ "_" ++ lists:append(Ids)). +columns_sql(lookup) -> "id, from_jid, message"; +columns_sql(count) -> "COUNT(*)". + query_type_to_id(lookup) -> "lookup"; query_type_to_id(count) -> "count". @@ -469,15 +488,6 @@ order_to_sql(asc) -> " ORDER BY id "; order_to_sql(desc) -> " ORDER BY id DESC "; order_to_sql(unordered) -> " ". -type_to_id(le) -> "le"; %% lower or equal -type_to_id(ge) -> "ge"; %% greater or equal -type_to_id(equal) -> "eq"; -type_to_id(lower) -> "lt"; %% lower than -type_to_id(greater) -> "gt"; %% greater than -type_to_id(like) -> "lk"; -type_to_id(limit) -> "li"; -type_to_id(offset) -> "of". - column_to_id(id) -> "i"; column_to_id(user_id) -> "u"; column_to_id(remote_bare_jid) -> "b"; @@ -498,35 +508,23 @@ skip_undefined(List) -> [X || X <- List, X =/= undefined]. filter_to_sql({Op, Column, _Value}) -> filter_to_sql(atom_to_list(Column), Op). -filter_to_sql(_Column, limit) -> undefined; -filter_to_sql(_Column, offset) -> undefined; +op_to_id(equal) -> "eq"; +op_to_id(lower) -> "lt"; %% lower than +op_to_id(greater) -> "gt"; %% greater than +op_to_id(le) -> "le"; %% lower or equal +op_to_id(ge) -> "ge"; %% greater or equal +op_to_id(like) -> "lk"; +op_to_id(limit) -> "li"; +op_to_id(offset) -> "of". + +filter_to_sql(Column, equal) -> Column ++ " = ?"; filter_to_sql(Column, lower) -> Column ++ " < ?"; filter_to_sql(Column, greater) -> Column ++ " > ?"; filter_to_sql(Column, le) -> Column ++ " <= ?"; filter_to_sql(Column, ge) -> Column ++ " >= ?"; -filter_to_sql(Column, equal) -> Column ++ " = ?"; -filter_to_sql(Column, like) -> Column ++ " LIKE ?". - -extend_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, - borders := Borders, search_text := SearchText} = Params, Env) -> - Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText), - start_id => make_start_id(Start, Borders), - end_id => make_end_id(End, Borders), - remote_bare_jid => maybe_encode_bare_jid(WithJID, Env), - remote_resource => jid_to_non_empty_resource(WithJID)}. - -jid_to_non_empty_resource(#jid{lresource = Res}) when byte_size(Res) > 0 -> Res; -jid_to_non_empty_resource(_) -> undefined. - --record(lookup_field, {op, column, param, required, value_maker}). - -lookup_fields() -> - [#lookup_field{op = equal, column = user_id, param = archive_id, required = true}, - #lookup_field{op = ge, column = id, param = start_id}, - #lookup_field{op = le, column = id, param = end_id}, - #lookup_field{op = equal, column = remote_bare_jid, param = remote_bare_jid}, - #lookup_field{op = equal, column = remote_resource, param = remote_resource}, - #lookup_field{op = like, column = search_body, param = norm_search_text, value_maker = search_words}]. +filter_to_sql(Column, like) -> Column ++ " LIKE ?"; +filter_to_sql(_Column, limit) -> undefined; +filter_to_sql(_Column, offset) -> undefined. prepare_filter(Params) -> [new_filter(Field, Value) @@ -544,7 +542,7 @@ field_to_values(#lookup_field{param = Param, value_maker = ValueMaker, required end. make_value(search_words, Value) -> search_words(Value); -make_value(undefined, Value) -> [Value]. +make_value(undefined, Value) -> [Value]. %% Default value_maker new_filter(#lookup_field{op = Op, column = Column}, Value) -> {Op, Column, Value}. @@ -552,6 +550,7 @@ new_filter(#lookup_field{op = Op, column = Column}, Value) -> %% Constructs a separate LIKE filter for each word. %% SearchText example is "word1%word2%word3". %% Order of words does not matter (they can go in any order). +-spec search_words(binary()) -> list(binary()). search_words(SearchText) -> Words = binary:split(SearchText, <<"%">>, [global]), [<<"%", Word/binary, "%">> || Word <- lists:sublist(Words, ?SEARCH_WORDS_LIMIT)]. From 57eb84849bcb97bf52f089bfb38e9597a473e858 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 15:03:57 +0100 Subject: [PATCH 27/92] Move unescape_binary into decode_packet --- src/mam/mod_mam_muc_rdbms_arch.erl | 26 +++++++++++++------------- src/mam/mod_mam_rdbms_arch.erl | 9 ++++----- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 4d33cdfb6e..6e1537bd10 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -71,6 +71,16 @@ start(Host, Opts) -> prepare_insert(insert_mam_muc_message, 1), + mongoose_rdbms:prepare(mam_muc_archive_size, mam_muc_message, [user_id], + [<<"SELECT COUNT(*) FROM mam_muc_message " + "WHERE room_id = ?">>]), + + mongoose_rdbms:prepare(mam_muc_extract_gdpr_messages, mam_muc_message, + [sender_id], + [<<"SELECT id, message " + " FROM mam_muc_message " + " WHERE sender_id=? " + " ORDER BY id">>]), start_muc(Host, Opts). -spec stop(jid:server()) -> 'ok'. @@ -116,13 +126,8 @@ stop_muc(Host) -> -spec archive_size(integer(), jid:server(), integer(), jid:jid()) -> integer(). archive_size(Size, Host, RoomID, _RoomJID) when is_integer(Size) -> - {selected, [{BSize}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) " - "FROM mam_muc_message ", - "WHERE room_id = ", use_escaped_integer(escape_room_id(RoomID))]), - mongoose_rdbms:result_to_integer(BSize). + Result = mod_mam_utils:success_sql_execute(Host, mam_muc_archive_size, [RoomID]), + mongoose_rdbms:selected_to_integer(Result). -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok. archive_message(_Result, Host, Params = #{direction := incoming}) -> @@ -465,12 +470,7 @@ do_extract_messages(Host, Filter, IOffset, IMax, Order) -> Filter, Order, LimitSQL, Offset]). extract_gdpr_messages(Host, ArchiveID) -> - Filter = ["WHERE sender_id = ", use_escaped_integer(escape_integer(ArchiveID))], - mod_mam_utils:success_sql_query( - Host, - ["SELECT id, message " - "FROM mam_muc_message ", - Filter, " ORDER BY id"]). + mod_mam_utils:success_sql_execute(Host, mam_muc_extract_gdpr_messages, [ArchiveID]). %% @doc Zero-based index of the row with UMessID in the result test. %% If the element does not exists, the MessID of the next element will diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 357e47fd55..642326a36c 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -241,8 +241,7 @@ make_tombstone(_Host, UserID, OriginID, [], _Env) -> ?LOG_INFO(#{what => make_tombstone_failed, text => <<"Message to retract was not found by origin id">>, user_id => UserID, origin_id => OriginID}); -make_tombstone(Host, UserID, OriginID, [{ResMessID, ResData}], Env) -> - Data = unescape_binary(ResData, Env), +make_tombstone(Host, UserID, OriginID, [{ResMessID, Data}], Env) -> Packet = decode_packet(Data, Env), MessID = mongoose_rdbms:result_to_integer(ResMessID), Tombstone = mod_mam_utils:tombstone(Packet, OriginID), @@ -311,10 +310,9 @@ lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> Filter = prepare_filter(ExtParams), mam_lookup:lookup(Env, Filter, ExtParams). -row_to_uniform_format({BMessID, BSrcJID, SDataRaw}, Env) -> +row_to_uniform_format({BMessID, BSrcJID, Data}, Env) -> MessID = mongoose_rdbms:result_to_integer(BMessID), SrcJID = decode_jid(BSrcJID, Env), - Data = unescape_binary(SDataRaw, Env), Packet = decode_packet(Data, Env), {MessID, SrcJID, Packet}. @@ -408,7 +406,8 @@ encode_packet(Packet, #{db_message_codec := Codec}) -> mam_message:encode(Codec, Packet). -spec decode_packet(binary(), env_vars()) -> exml:element(). -decode_packet(Bin, #{db_message_codec := Codec}) -> +decode_packet(EncBin, #{db_message_codec := Codec}) -> + Bin = unescape_binary(EncBin, Env), mam_message:decode(Codec, Bin). -spec unescape_binary(binary(), env_vars()) -> binary(). From bdb4944d837ef94e2145effbd09c01292ce61a85 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 17:37:55 +0100 Subject: [PATCH 28/92] Move some code into mam_filter and mam_lookup_sql --- include/mongoose_mam.hrl | 2 + src/mam/mam_filter.erl | 35 ++++++ src/mam/mam_lookup.erl | 19 ++-- src/mam/mam_lookup_sql.erl | 110 +++++++++++++++++++ src/mam/mod_mam_rdbms_arch.erl | 190 +++++---------------------------- 5 files changed, 186 insertions(+), 170 deletions(-) create mode 100644 include/mongoose_mam.hrl create mode 100644 src/mam/mam_filter.erl create mode 100644 src/mam/mam_lookup_sql.erl diff --git a/include/mongoose_mam.hrl b/include/mongoose_mam.hrl new file mode 100644 index 0000000000..4a34faba7c --- /dev/null +++ b/include/mongoose_mam.hrl @@ -0,0 +1,2 @@ +-record(db_mapping, {column, param, format}). +-record(lookup_field, {op, column, param, required, value_maker}). diff --git a/src/mam/mam_filter.erl b/src/mam/mam_filter.erl new file mode 100644 index 0000000000..40ca9a0891 --- /dev/null +++ b/src/mam/mam_filter.erl @@ -0,0 +1,35 @@ +%% Produces filters based on lookup params +-module(mam_filter). +-export([produce_filter/2]). +-include("mongoose_mam.hrl"). + +-define(SEARCH_WORDS_LIMIT, 10). + +produce_filter(Params, Fields) -> + [new_filter(Field, Value) + || Field <- Fields, + Value <- field_to_values(Field, Params)]. + +field_to_values(#lookup_field{param = Param, value_maker = ValueMaker, required = Required} = Field, Params) -> + case maps:find(Param, Params) of + {ok, Value} when Value =/= undefined -> + make_value(ValueMaker, Value); + Other when Required -> + error(#{reason => missing_required_field, field => Field, params => Params, result => Other}); + _ -> + [] + end. + +make_value(search_words, Value) -> search_words(Value); +make_value(undefined, Value) -> [Value]. %% Default value_maker + +new_filter(#lookup_field{op = Op, column = Column}, Value) -> + {Op, Column, Value}. + +%% Constructs a separate LIKE filter for each word. +%% SearchText example is "word1%word2%word3". +%% Order of words does not matter (they can go in any order). +-spec search_words(binary()) -> list(binary()). +search_words(SearchText) -> + Words = binary:split(SearchText, <<"%">>, [global]), + [<<"%", Word/binary, "%">> || Word <- lists:sublist(Words, ?SEARCH_WORDS_LIMIT)]. diff --git a/src/mam/mam_lookup.erl b/src/mam/mam_lookup.erl index 18fa6c00fd..c297c77f8d 100644 --- a/src/mam/mam_lookup.erl +++ b/src/mam/mam_lookup.erl @@ -16,7 +16,7 @@ %% Public logic %% We use two fields from Env: %% - lookup_fun -%% - row_to_uniform_format_fun +%% - decode_row_fun -spec lookup(env_vars(), filter(), params()) -> {ok, mod_mam:lookup_result()} | {error, item_not_found}. lookup(Env = #{}, Filter, Params = #{rsm := RSM}) when is_list(Filter) -> @@ -26,8 +26,8 @@ lookup(Env = #{}, Filter, Params = #{rsm := RSM}) when is_list(Filter) -> lookup_query(QueryType, #{lookup_fun := LookupF} = Env, Filters, Order) -> LookupF(QueryType, Env, Filters, Order). -row_to_uniform_format(Row, #{row_to_uniform_format_fun := FormatF} = Env) -> - FormatF(Row, Env). +decode_row(Row, #{decode_row_fun := DecodeF} = Env) -> + DecodeF(Row, Env). %% Private logic below @@ -101,7 +101,7 @@ lookup_last_page(Env, PageSize, Filter) -> true -> 0; %% Result fits on a single page false -> - FirstID = uniform_to_message_id(hd(Messages)), + FirstID = decoded_row_to_message_id(hd(Messages)), calc_count(Env, before_id(FirstID, Filter)) end, {ok, {Offset + Selected, Offset, Messages}}. @@ -116,7 +116,7 @@ lookup_by_offset(Env, RSM, PageSize, Filter) -> true -> Offset + Selected; %% Result fits on a single page false -> - LastID = uniform_to_message_id(lists:last(Messages)), + LastID = decoded_row_to_message_id(lists:last(Messages)), CountAfterLastID = calc_count(Env, after_id(LastID, Filter)), Offset + Selected + CountAfterLastID end, @@ -154,10 +154,11 @@ rsm_to_regular_lookup_vars(RSM, Filter, Offset, PageSize) -> {false, Filter, Offset, PageSize, asc} end. -rows_to_uniform_format(MessageRows, Env) -> - [row_to_uniform_format(Row, Env) || Row <- MessageRows]. +decode_rows(MessageRows, Env) -> + [decode_row(Row, Env) || Row <- MessageRows]. -uniform_to_message_id({MessID, _, _}) -> MessID. +%% First element of the tuple is decoded message ID +decoded_row_to_message_id(DecodedRow) -> element(1, DecodedRow). -spec extract_messages(Env :: env_vars(), Filter :: filter(), Offset :: non_neg_integer(), Max :: pos_integer(), @@ -167,7 +168,7 @@ extract_messages(_Env, _Filter, _Offset, 0 = _Max, _Order) -> extract_messages(Env, Filter, Offset, Max, Order) -> {selected, MessageRows} = extract_rows(Env, Filter, Offset, Max, Order), Rows = maybe_reverse(Order, MessageRows), - rows_to_uniform_format(Rows, Env). + decode_rows(Rows, Env). maybe_reverse(asc, List) -> List; maybe_reverse(desc, List) -> lists:reverse(List). diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl new file mode 100644 index 0000000000..0128fcadb5 --- /dev/null +++ b/src/mam/mam_lookup_sql.erl @@ -0,0 +1,110 @@ +%% Makes a SELECT SQL query +-module(mam_lookup_sql). +-export([lookup_query/4]). + +-include("mongoose_logger.hrl"). +-include("mongoose_mam.hrl"). + +lookup_query(QueryType, #{host := Host, table := Table} = Env, Filters, Order) -> + StmtName = filters_to_statement_name(Env, QueryType, Filters, Order), + case mongoose_rdbms:prepared(StmtName) of + false -> + %% Create a new type of a query + SQL = lookup_sql_binary(QueryType, Env, Filters, Order), + Columns = filters_to_columns(Filters), + mongoose_rdbms:prepare(StmtName, Table, Columns, SQL); + true -> + ok + end, + Args = filters_to_args(Filters), + try mongoose_rdbms:execute(Host, StmtName, Args) of + {selected, Rs} when is_list(Rs) -> + {selected, Rs}; + Error -> + What = #{what => mam_lookup_failed, statement => StmtName, + sql_query => lookup_sql_binary(QueryType, Env, Filters, Order), + reason => Error, host => Host}, + ?LOG_ERROR(What), + error(What) + catch Class:Error:Stacktrace -> + What = #{what => mam_lookup_failed, statement => StmtName, + sql_query => lookup_sql_binary(QueryType, Env, Filters, Order), + class => Class, stacktrace => Stacktrace, + reason => Error, host => Host}, + ?LOG_ERROR(What), + erlang:raise(Class, Error, Stacktrace) + end. + +lookup_sql_binary(QueryType, Env, Filters, Order) -> + iolist_to_binary(lookup_sql(QueryType, Env, Filters, Order)). + +lookup_sql(QueryType, Env = #{index_hint_fn := IndexHintF, table := Table}, Filters, Order) -> + IndexHintSQL = IndexHintF(Env), + FilterSQL = filters_to_sql(Filters), + OrderSQL = order_to_sql(Order), + LimitSQL = limit_sql(Filters), + ["SELECT ", columns_sql(Env, QueryType), " FROM ", atom_to_list(Table), " ", + IndexHintSQL, FilterSQL, OrderSQL, LimitSQL]. + +%% Caller should provide both limit and offset fields in the correct order. +%% See limit_offset_filters. +%% No limits option is fine too (it is used with count and GDPR). +limit_sql(Filters) -> + HasLimit = lists:keymember(limit, 1, Filters), %% and offset + case HasLimit of + true -> rdbms_queries:limit_offset_sql(); + false -> "" + end. + +filters_to_columns(Filters) -> + [Column || {_Op, Column, _Value} <- Filters]. + +filters_to_args(Filters) -> + [Value || {_Op, _Column, Value} <- Filters]. + +filters_to_statement_name(#{table := Table, column_to_id_fn := ColFn}, QueryType, Filters, Order) -> + QueryId = query_type_to_id(QueryType), + Ids = [op_to_id(Op) ++ ColFn(Col) || {Op, Col, _Val} <- Filters], + OrderId = order_type_to_id(Order), + list_to_atom(atom_to_list(Table) ++ "_" ++ QueryId ++ "_" ++ OrderId ++ "_" ++ lists:append(Ids)). + +columns_sql(#{columns_sql_fn := F}, QueryType) -> F(QueryType). + +query_type_to_id(QueryType) -> atom_to_list(QueryType). + +order_type_to_id(desc) -> "d"; +order_type_to_id(asc) -> "a"; +order_type_to_id(unordered) -> "u". + +order_to_sql(asc) -> " ORDER BY id "; +order_to_sql(desc) -> " ORDER BY id DESC "; +order_to_sql(unordered) -> " ". + +filters_to_sql(Filters) -> + SQLs = [filter_to_sql(Filter) || Filter <- Filters], + case skip_undefined(SQLs) of + [] -> ""; + Defined -> [" WHERE ", rdbms_queries:join(Defined, " AND ")] + end. + +skip_undefined(List) -> [X || X <- List, X =/= undefined]. + +filter_to_sql({Op, Column, _Value}) -> filter_to_sql(atom_to_list(Column), Op). + +op_to_id(equal) -> "eq"; +op_to_id(lower) -> "lt"; %% lower than +op_to_id(greater) -> "gt"; %% greater than +op_to_id(le) -> "le"; %% lower or equal +op_to_id(ge) -> "ge"; %% greater or equal +op_to_id(like) -> "lk"; +op_to_id(limit) -> "li"; +op_to_id(offset) -> "of". + +filter_to_sql(Column, equal) -> Column ++ " = ?"; +filter_to_sql(Column, lower) -> Column ++ " < ?"; +filter_to_sql(Column, greater) -> Column ++ " > ?"; +filter_to_sql(Column, le) -> Column ++ " <= ?"; +filter_to_sql(Column, ge) -> Column ++ " >= ?"; +filter_to_sql(Column, like) -> Column ++ " LIKE ?"; +filter_to_sql(_Column, limit) -> undefined; +filter_to_sql(_Column, offset) -> undefined. diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 642326a36c..5d12a01130 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -6,8 +6,6 @@ %%%------------------------------------------------------------------- -module(mod_mam_rdbms_arch). --define(SEARCH_WORDS_LIMIT, 10). - %% ---------------------------------------------------------------------- %% Exports @@ -48,6 +46,7 @@ -include("jlib.hrl"). -include_lib("exml/include/exml.hrl"). -include("mongoose_rsm.hrl"). +-include("mongoose_mam.hrl"). %% ---------------------------------------------------------------------- %% Types @@ -68,9 +67,6 @@ -type value_type() :: int | maybe_string | direction | bare_jid | jid | jid_resource | xml | search. --record(db_mapping, {column, param, format}). --record(lookup_field, {op, column, param, required, value_maker}). - db_mappings() -> [#db_mapping{column = id, param = message_id, format = int}, #db_mapping{column = user_id, param = archive_id, format = int}, @@ -95,8 +91,12 @@ env_vars(Host, ArcJID) -> %% It's only for passing into RDBMS. #{host => Host, archive_jid => ArcJID, + table => mam_message, + index_hint_fn => fun index_hint_sql/1, + columns_sql_fn => fun columns_sql/1, + column_to_id_fn => fun column_to_id/1, lookup_fun => fun lookup_query/4, - row_to_uniform_format_fun => fun row_to_uniform_format/2, + decode_row_fun => fun row_to_uniform_format/2, has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam, Host), full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), db_jid_codec => db_jid_codec(Host, ?MODULE), @@ -110,6 +110,25 @@ extend_lookup_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, remote_bare_jid => maybe_encode_bare_jid(WithJID, Env), remote_resource => jid_to_non_empty_resource(WithJID)}. +-spec index_hint_sql(env_vars()) -> string(). +index_hint_sql(#{host := Host}) -> + case mongoose_rdbms:db_engine(Host) of + mysql -> "USE INDEX(PRIMARY, i_mam_message_rem) "; + _ -> "" + end. + +columns_sql(lookup) -> "id, from_jid, message"; +columns_sql(count) -> "COUNT(*)". + +column_to_id(id) -> "i"; +column_to_id(user_id) -> "u"; +column_to_id(remote_bare_jid) -> "b"; +column_to_id(remote_resource) -> "r"; +column_to_id(search_body) -> "s"; +%% fictional columns +column_to_id(limit) -> "l"; +column_to_id(offset) -> "o". + %% ---------------------------------------------------------------------- %% gen_mod callbacks %% Starting and stopping functions for users' archives @@ -307,7 +326,7 @@ lookup_messages({error, _Reason}=Result, _Host, _Params) -> lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), ExtParams = extend_lookup_params(Params, Env), - Filter = prepare_filter(ExtParams), + Filter = mam_filter:produce_filter(ExtParams, lookup_fields()), mam_lookup:lookup(Env, Filter, ExtParams). row_to_uniform_format({BMessID, BSrcJID, Data}, Env) -> @@ -316,36 +335,8 @@ row_to_uniform_format({BMessID, BSrcJID, Data}, Env) -> Packet = decode_packet(Data, Env), {MessID, SrcJID, Packet}. -lookup_query(QueryType, #{host := Host} = _Env, Filters, Order) -> - StmtName = filters_to_statement_name(QueryType, Filters, Order), - case mongoose_rdbms:prepared(StmtName) of - false -> - %% Create a new type of a query - SQL = lookup_sql_binary(QueryType, Filters, Order, index_hint_sql(Host)), - Columns = filters_to_columns(Filters), - mongoose_rdbms:prepare(StmtName, mam_message, Columns, SQL); - true -> - ok - end, - Args = filters_to_args(Filters), - try mongoose_rdbms:execute(Host, StmtName, Args) of - {selected, Rs} when is_list(Rs) -> - {selected, Rs}; - Error -> - What = #{what => mam_lookup_failed, statement => StmtName, - sql_query => lookup_sql_binary(QueryType, Filters, Order, index_hint_sql(Host)), - reason => Error, host => Host}, - ?LOG_ERROR(What), - error(What) - catch Class:Error:Stacktrace -> - What = #{what => mam_lookup_failed, statement => StmtName, - sql_query => lookup_sql_binary(QueryType, Filters, Order, index_hint_sql(Host)), - class => Class, stacktrace => Stacktrace, - reason => Error, host => Host}, - ?LOG_ERROR(What), - error(What) - end. - +lookup_query(QueryType, Env, Filters, Order) -> + mam_lookup_sql:lookup_query(QueryType, Env, Filters, Order). %% ---------------------------------------------------------------------- %% Optimizations and extensible code @@ -406,7 +397,7 @@ encode_packet(Packet, #{db_message_codec := Codec}) -> mam_message:encode(Codec, Packet). -spec decode_packet(binary(), env_vars()) -> exml:element(). -decode_packet(EncBin, #{db_message_codec := Codec}) -> +decode_packet(EncBin, Env = #{db_message_codec := Codec}) -> Bin = unescape_binary(EncBin, Env), mam_message:decode(Codec, Bin). @@ -430,126 +421,3 @@ db_jid_codec(Host, Module) -> -spec db_message_codec(jid:server(), module()) -> module(). db_message_codec(Host, Module) -> gen_mod:get_module_opt(Host, Module, db_message_format, mam_message_compressed_eterm). - -%% ---------------------------------------------------------------------- -%% Prepared queries helpers - -lookup_sql_binary(QueryType, Filters, Order, IndexHintSQL) -> - iolist_to_binary(lookup_sql(QueryType, Filters, Order, IndexHintSQL)). - -lookup_sql(QueryType, Filters, Order, IndexHintSQL) -> - LimitSQL = limit_sql(Filters), - OrderSQL = order_to_sql(Order), - FilterSQL = filters_to_sql(Filters), - ["SELECT ", columns_sql(QueryType), " FROM mam_message ", - IndexHintSQL, FilterSQL, OrderSQL, LimitSQL]. - -%% Caller should provide both limit and offset fields in the correct order. -%% See limit_offset_filters. -%% No limits option is fine too (it is used with count and GDPR). -limit_sql(Filters) -> - HasLimit = lists:keymember(limit, 1, Filters), %% and offset - case HasLimit of - true -> rdbms_queries:limit_offset_sql(); - false -> "" - end. - --spec index_hint_sql(jid:server()) -> string(). -index_hint_sql(Host) -> - case mongoose_rdbms:db_engine(Host) of - mysql -> "USE INDEX(PRIMARY, i_mam_message_rem) "; - _ -> "" - end. - -filters_to_columns(Filters) -> - [Column || {_Op, Column, _Value} <- Filters]. - -filters_to_args(Filters) -> - [Value || {_Op, _Column, Value} <- Filters]. - -filters_to_statement_name(QueryType, Filters, Order) -> - QueryId = query_type_to_id(QueryType), - Ids = [op_to_id(Op) ++ column_to_id(Col) || {Op, Col, _Val} <- Filters], - OrderId = order_type_to_id(Order), - list_to_atom("mam_" ++ QueryId ++ "_" ++ OrderId ++ "_" ++ lists:append(Ids)). - -columns_sql(lookup) -> "id, from_jid, message"; -columns_sql(count) -> "COUNT(*)". - -query_type_to_id(lookup) -> "lookup"; -query_type_to_id(count) -> "count". - -order_type_to_id(desc) -> "d"; -order_type_to_id(asc) -> "a"; -order_type_to_id(unordered) -> "u". - -order_to_sql(asc) -> " ORDER BY id "; -order_to_sql(desc) -> " ORDER BY id DESC "; -order_to_sql(unordered) -> " ". - -column_to_id(id) -> "i"; -column_to_id(user_id) -> "u"; -column_to_id(remote_bare_jid) -> "b"; -column_to_id(remote_resource) -> "r"; -column_to_id(search_body) -> "s"; -%% fictional columns -column_to_id(limit) -> "l"; -column_to_id(offset) -> "o". - -filters_to_sql(Filters) -> - SQLs = [filter_to_sql(Filter) || Filter <- Filters], - case skip_undefined(SQLs) of - [] -> ""; - Defined -> [" WHERE ", rdbms_queries:join(Defined, " AND ")] - end. - -skip_undefined(List) -> [X || X <- List, X =/= undefined]. - -filter_to_sql({Op, Column, _Value}) -> filter_to_sql(atom_to_list(Column), Op). - -op_to_id(equal) -> "eq"; -op_to_id(lower) -> "lt"; %% lower than -op_to_id(greater) -> "gt"; %% greater than -op_to_id(le) -> "le"; %% lower or equal -op_to_id(ge) -> "ge"; %% greater or equal -op_to_id(like) -> "lk"; -op_to_id(limit) -> "li"; -op_to_id(offset) -> "of". - -filter_to_sql(Column, equal) -> Column ++ " = ?"; -filter_to_sql(Column, lower) -> Column ++ " < ?"; -filter_to_sql(Column, greater) -> Column ++ " > ?"; -filter_to_sql(Column, le) -> Column ++ " <= ?"; -filter_to_sql(Column, ge) -> Column ++ " >= ?"; -filter_to_sql(Column, like) -> Column ++ " LIKE ?"; -filter_to_sql(_Column, limit) -> undefined; -filter_to_sql(_Column, offset) -> undefined. - -prepare_filter(Params) -> - [new_filter(Field, Value) - || Field <- lookup_fields(), - Value <- field_to_values(Field, Params)]. - -field_to_values(#lookup_field{param = Param, value_maker = ValueMaker, required = Required} = Field, Params) -> - case maps:find(Param, Params) of - {ok, Value} when Value =/= undefined -> - make_value(ValueMaker, Value); - Other when Required -> - error(#{reason => missing_required_field, field => Field, params => Params, result => Other}); - _ -> - [] - end. - -make_value(search_words, Value) -> search_words(Value); -make_value(undefined, Value) -> [Value]. %% Default value_maker - -new_filter(#lookup_field{op = Op, column = Column}, Value) -> - {Op, Column, Value}. - -%% Constructs a separate LIKE filter for each word. -%% SearchText example is "word1%word2%word3". -%% Order of words does not matter (they can go in any order). --spec search_words(binary()) -> list(binary()). -search_words(SearchText) -> - Words = binary:split(SearchText, <<"%">>, [global]), - [<<"%", Word/binary, "%">> || Word <- lists:sublist(Words, ?SEARCH_WORDS_LIMIT)]. From 52b64a30a829a10a6599363cd6eb78f9589342e2 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 17:52:14 +0100 Subject: [PATCH 29/92] Rename some env fields --- src/mam/mam_lookup.erl | 8 ++++---- src/mam/mam_lookup_sql.erl | 11 +++++++++++ src/mam/mod_mam_rdbms_arch.erl | 8 ++++---- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/mam/mam_lookup.erl b/src/mam/mam_lookup.erl index c297c77f8d..82261a6d3a 100644 --- a/src/mam/mam_lookup.erl +++ b/src/mam/mam_lookup.erl @@ -15,18 +15,18 @@ %% Public logic %% We use two fields from Env: -%% - lookup_fun -%% - decode_row_fun +%% - lookup_fn +%% - decode_row_fn -spec lookup(env_vars(), filter(), params()) -> {ok, mod_mam:lookup_result()} | {error, item_not_found}. lookup(Env = #{}, Filter, Params = #{rsm := RSM}) when is_list(Filter) -> OptParams = Params#{opt_count_type => opt_count_type(RSM)}, choose_lookup_messages_strategy(Env, Filter, OptParams). -lookup_query(QueryType, #{lookup_fun := LookupF} = Env, Filters, Order) -> +lookup_query(QueryType, #{lookup_fn := LookupF} = Env, Filters, Order) -> LookupF(QueryType, Env, Filters, Order). -decode_row(Row, #{decode_row_fun := DecodeF} = Env) -> +decode_row(Row, #{decode_row_fn := DecodeF} = Env) -> DecodeF(Row, Env). %% Private logic below diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index 0128fcadb5..8e84479356 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -5,6 +5,17 @@ -include("mongoose_logger.hrl"). -include("mongoose_mam.hrl"). + +%% This function uses some fields from Env: +%% - host +%% - table +%% - index_hint_fn +%% - column_to_id_fn +%% - columns_sql_fn +%% +%% Filters are in format {Op, Column, Value} +%% QueryType should be an atom, that we pass into the columns_sql_fn function. +-spec lookup_query(QueryType :: atom(), Env :: map(), Filters :: list(), Order :: atom()) -> term(). lookup_query(QueryType, #{host := Host, table := Table} = Env, Filters, Order) -> StmtName = filters_to_statement_name(Env, QueryType, Filters, Order), case mongoose_rdbms:prepared(StmtName) of diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 5d12a01130..228361490e 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -95,10 +95,10 @@ env_vars(Host, ArcJID) -> index_hint_fn => fun index_hint_sql/1, columns_sql_fn => fun columns_sql/1, column_to_id_fn => fun column_to_id/1, - lookup_fun => fun lookup_query/4, - decode_row_fun => fun row_to_uniform_format/2, + lookup_fn => fun lookup_query/4, + decode_row_fn => fun row_to_uniform_format/2, has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam, Host), - full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), + has_full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), db_jid_codec => db_jid_codec(Host, ?MODULE), db_message_codec => db_message_codec(Host, ?MODULE)}. @@ -411,7 +411,7 @@ get_retract_id(Packet, #{has_message_retraction := Enabled}) -> mod_mam_utils:get_retract_id(Enabled, Packet). -spec encode_search_body(exml:element(), env_vars()) -> binary(). -encode_search_body(Packet, #{full_text_search := SearchEnabled}) -> +encode_search_body(Packet, #{has_full_text_search := SearchEnabled}) -> mod_mam_utils:packet_to_search_body(SearchEnabled, Packet). -spec db_jid_codec(jid:server(), module()) -> module(). From fdaec9d19913c96e09cf2a760d1d82c9541a7a26 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 18:10:10 +0100 Subject: [PATCH 30/92] Put env getters on top of mam_lookup_sql --- src/mam/mam_lookup_sql.erl | 35 ++++++++++++++++++++-------------- src/mam/mod_mam_rdbms_arch.erl | 4 +--- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index 8e84479356..2ce982b0d1 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -5,23 +5,32 @@ -include("mongoose_logger.hrl"). -include("mongoose_mam.hrl"). +%% The ONLY usage of Env is in these functions: +%% The rest of code should treat Env as opaque (i.e. the code just passes Env around). +host(#{host := Host}) -> Host. +table(#{table := Table}) -> Table. +index_hint_sql(Env = #{index_hint_fn := F}) -> F(Env). +columns_sql(#{columns_sql_fn := F}, QueryType) -> F(QueryType). +column_to_id(#{column_to_id_fn := F}, Col) -> F(Col). %% This function uses some fields from Env: %% - host %% - table %% - index_hint_fn -%% - column_to_id_fn %% - columns_sql_fn +%% - column_to_id_fn %% %% Filters are in format {Op, Column, Value} %% QueryType should be an atom, that we pass into the columns_sql_fn function. -spec lookup_query(QueryType :: atom(), Env :: map(), Filters :: list(), Order :: atom()) -> term(). -lookup_query(QueryType, #{host := Host, table := Table} = Env, Filters, Order) -> - StmtName = filters_to_statement_name(Env, QueryType, Filters, Order), +lookup_query(QueryType, Env, Filters, Order) -> + Table = table(Env), + Host = host(Env), + StmtName = filters_to_statement_name(Env, QueryType, Table, Filters, Order), case mongoose_rdbms:prepared(StmtName) of false -> %% Create a new type of a query - SQL = lookup_sql_binary(QueryType, Env, Filters, Order), + SQL = lookup_sql_binary(QueryType, Table, Env, Filters, Order), Columns = filters_to_columns(Filters), mongoose_rdbms:prepare(StmtName, Table, Columns, SQL); true -> @@ -33,24 +42,24 @@ lookup_query(QueryType, #{host := Host, table := Table} = Env, Filters, Order) - {selected, Rs}; Error -> What = #{what => mam_lookup_failed, statement => StmtName, - sql_query => lookup_sql_binary(QueryType, Env, Filters, Order), + sql_query => lookup_sql_binary(QueryType, Table, Env, Filters, Order), reason => Error, host => Host}, ?LOG_ERROR(What), error(What) catch Class:Error:Stacktrace -> What = #{what => mam_lookup_failed, statement => StmtName, - sql_query => lookup_sql_binary(QueryType, Env, Filters, Order), + sql_query => lookup_sql_binary(QueryType, Table, Env, Filters, Order), class => Class, stacktrace => Stacktrace, reason => Error, host => Host}, ?LOG_ERROR(What), erlang:raise(Class, Error, Stacktrace) end. -lookup_sql_binary(QueryType, Env, Filters, Order) -> - iolist_to_binary(lookup_sql(QueryType, Env, Filters, Order)). +lookup_sql_binary(QueryType, Table, Env, Filters, Order) -> + iolist_to_binary(lookup_sql(QueryType, Table, Env, Filters, Order)). -lookup_sql(QueryType, Env = #{index_hint_fn := IndexHintF, table := Table}, Filters, Order) -> - IndexHintSQL = IndexHintF(Env), +lookup_sql(QueryType, Table, Env, Filters, Order) -> + IndexHintSQL = index_hint_sql(Env), FilterSQL = filters_to_sql(Filters), OrderSQL = order_to_sql(Order), LimitSQL = limit_sql(Filters), @@ -73,14 +82,12 @@ filters_to_columns(Filters) -> filters_to_args(Filters) -> [Value || {_Op, _Column, Value} <- Filters]. -filters_to_statement_name(#{table := Table, column_to_id_fn := ColFn}, QueryType, Filters, Order) -> +filters_to_statement_name(Env, QueryType, Table, Filters, Order) -> QueryId = query_type_to_id(QueryType), - Ids = [op_to_id(Op) ++ ColFn(Col) || {Op, Col, _Val} <- Filters], + Ids = [op_to_id(Op) ++ column_to_id(Env, Col) || {Op, Col, _Val} <- Filters], OrderId = order_type_to_id(Order), list_to_atom(atom_to_list(Table) ++ "_" ++ QueryId ++ "_" ++ OrderId ++ "_" ++ lists:append(Ids)). -columns_sql(#{columns_sql_fn := F}, QueryType) -> F(QueryType). - query_type_to_id(QueryType) -> atom_to_list(QueryType). order_type_to_id(desc) -> "d"; diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 228361490e..1e5814e38e 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -133,10 +133,9 @@ column_to_id(offset) -> "o". %% gen_mod callbacks %% Starting and stopping functions for users' archives --spec start(jid:server(), _) -> 'ok'. +-spec start(jid:server(), _) -> ok. start(Host, Opts) -> start_pm(Host, Opts), - prepare_insert(insert_mam_message, 1), mongoose_rdbms:prepare(mam_archive_remove, mam_message, [user_id], [<<"DELETE FROM mam_message " @@ -144,7 +143,6 @@ start(Host, Opts) -> mongoose_rdbms:prepare(mam_make_tombstone, mam_message, [message, user_id, id], [<<"UPDATE mam_message SET message = ?, search_body = '' " "WHERE user_id = ? AND id = ?">>]), - {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(1), mongoose_rdbms:prepare(mam_select_messages_to_retract, mam_message, [user_id, remote_bare_jid, origin_id, direction], From 0228e7202e17c3f72ba79f8dd1c0d265c69a055d Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 18:29:36 +0100 Subject: [PATCH 31/92] Split make_tombstone function --- src/mam/mod_mam_rdbms_arch.erl | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 1e5814e38e..539a378564 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -243,24 +243,29 @@ retract_message(Host, #{archive_id := UserID, remote_jid := RemJID, direction := Dir, packet := Packet}, Env) -> case get_retract_id(Packet, Env) of none -> ok; - OriginID -> retract_message(Host, UserID, RemJID, OriginID, Dir, Env) + OriginID -> + Info = get_retraction_info(Host, UserID, RemJID, OriginID, Dir, Env), + make_tombstone(Host, UserID, OriginID, Info, Env) end. -retract_message(Host, UserID, RemJID, OriginID, Dir, Env) -> +get_retraction_info(Host, UserID, RemJID, OriginID, Dir, Env) -> BinBareRemJID = encode_jid(jid:to_bare(RemJID), Env), BinDir = encode_direction(Dir), {selected, Rows} = execute_select_messages_to_retract( Host, UserID, BinBareRemJID, OriginID, BinDir), - make_tombstone(Host, UserID, OriginID, Rows, Env), - ok. + decode_retraction_info(Env, Rows). + +decode_retraction_info(_Env, []) -> skip; +decode_retraction_info(Env, [{ResMessID, Data}]) -> + Packet = decode_packet(Data, Env), + MessID = mongoose_rdbms:result_to_integer(ResMessID), + #{packet => Packet, message_id => MessID}. -make_tombstone(_Host, UserID, OriginID, [], _Env) -> +make_tombstone(_Host, UserID, OriginID, skip, _Env) -> ?LOG_INFO(#{what => make_tombstone_failed, text => <<"Message to retract was not found by origin id">>, user_id => UserID, origin_id => OriginID}); -make_tombstone(Host, UserID, OriginID, [{ResMessID, Data}], Env) -> - Packet = decode_packet(Data, Env), - MessID = mongoose_rdbms:result_to_integer(ResMessID), +make_tombstone(Host, UserID, OriginID, #{packet := Packet, message_id := MessID}, Env) -> Tombstone = mod_mam_utils:tombstone(Packet, OriginID), TombstoneData = encode_packet(Tombstone, Env), execute_make_tombstone(Host, TombstoneData, UserID, MessID). From 526aae62800b279ef114268ebc7efa49e2ded1aa Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 18:48:19 +0100 Subject: [PATCH 32/92] Move some code into mam_encoder --- src/mam/mam_encoder.erl | 84 ++++++++++++++++++++++++++++++ src/mam/mod_mam_rdbms_arch.erl | 93 +++------------------------------- 2 files changed, 90 insertions(+), 87 deletions(-) create mode 100644 src/mam/mam_encoder.erl diff --git a/src/mam/mam_encoder.erl b/src/mam/mam_encoder.erl new file mode 100644 index 0000000000..71c534f08b --- /dev/null +++ b/src/mam/mam_encoder.erl @@ -0,0 +1,84 @@ +-module(mam_encoder). +-export([encode_message/3]). +-export([encode_jid/2]). +-export([encode_direction/1]). +-export([encode_packet/2]). +-export([extend_lookup_params/2]). + +-type value_type() :: int | maybe_string | direction | bare_jid | jid | jid_resource | xml | search. +-type env_vars() :: map(). + +-include("mongoose.hrl"). +-include("jlib.hrl"). +-include_lib("exml/include/exml.hrl"). +-include("mongoose_mam.hrl"). + +extend_lookup_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, + borders := Borders, search_text := SearchText} = Params, Env) -> + Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText), + start_id => make_start_id(Start, Borders), + end_id => make_end_id(End, Borders), + remote_bare_jid => maybe_encode_bare_jid(WithJID, Env), + remote_resource => jid_to_non_empty_resource(WithJID)}. + +encode_message(Params, Env, Mappings) -> + [encode_value_using_mapping(Params, Env, Mapping) || Mapping <- Mappings]. + +encode_value_using_mapping(Params, Env, #db_mapping{param = Param, format = Format}) -> + Value = maps:get(Param, Params), + encode_value(Format, Value, Env). + +-spec encode_value(value_type(), term(), env_vars()) -> term(). +encode_value(int, Value, _Env) when is_integer(Value) -> + Value; +encode_value(maybe_string, none, _Env) -> + null; +encode_value(maybe_string, Value, _Env) when is_binary(Value) -> + Value; +encode_value(direction, Value, _Env) -> + encode_direction(Value); +encode_value(bare_jid, Value, Env) -> + encode_jid(jid:to_bare(Value), Env); +encode_value(jid, Value, Env) -> + encode_jid(Value, Env); +encode_value(jid_resource, #jid{lresource = Res}, _Env) -> + Res; +encode_value(xml, Value, Env) -> + encode_packet(Value, Env); +encode_value(search, Value, Env) -> + encode_search_body(Value, Env). + +encode_direction(incoming) -> <<"I">>; +encode_direction(outgoing) -> <<"O">>. + +make_start_id(Start, Borders) -> + StartID = maybe_encode_compact_uuid(Start, 0), + mod_mam_utils:apply_start_border(Borders, StartID). + +make_end_id(End, Borders) -> + EndID = maybe_encode_compact_uuid(End, 255), + mod_mam_utils:apply_end_border(Borders, EndID). + +maybe_encode_compact_uuid(undefined, _) -> + undefined; +maybe_encode_compact_uuid(Microseconds, NodeID) -> + mod_mam_utils:encode_compact_uuid(Microseconds, NodeID). + +jid_to_non_empty_resource(undefined) -> undefined; +jid_to_non_empty_resource(#jid{lresource = <<>>}) -> undefined; +jid_to_non_empty_resource(#jid{lresource = Res}) -> Res. + +-spec encode_jid(jid:jid(), env_vars()) -> binary(). +encode_jid(JID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> + mam_jid:encode(Codec, ArcJID, JID). + +maybe_encode_bare_jid(undefined, _Env) -> undefined; +maybe_encode_bare_jid(JID, Env) -> encode_jid(jid:to_bare(JID), Env). + +-spec encode_packet(exml:element(), env_vars()) -> binary(). +encode_packet(Packet, #{db_message_codec := Codec}) -> + mam_message:encode(Codec, Packet). + +-spec encode_search_body(exml:element(), env_vars()) -> binary(). +encode_search_body(Packet, #{has_full_text_search := SearchEnabled}) -> + mod_mam_utils:packet_to_search_body(SearchEnabled, Packet). diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 539a378564..729a81f5b8 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -33,15 +33,6 @@ %% ---------------------------------------------------------------------- %% Imports -%% UID --import(mod_mam_utils, - [encode_compact_uuid/2]). - -%% Other --import(mod_mam_utils, - [apply_start_border/2, - apply_end_border/2]). - -include("mongoose.hrl"). -include("jlib.hrl"). -include_lib("exml/include/exml.hrl"). @@ -65,8 +56,6 @@ -type env_vars() :: map(). --type value_type() :: int | maybe_string | direction | bare_jid | jid | jid_resource | xml | search. - db_mappings() -> [#db_mapping{column = id, param = message_id, format = int}, #db_mapping{column = user_id, param = archive_id, format = int}, @@ -102,14 +91,6 @@ env_vars(Host, ArcJID) -> db_jid_codec => db_jid_codec(Host, ?MODULE), db_message_codec => db_message_codec(Host, ?MODULE)}. -extend_lookup_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, - borders := Borders, search_text := SearchText} = Params, Env) -> - Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText), - start_id => make_start_id(Start, Borders), - end_id => make_end_id(End, Borders), - remote_bare_jid => maybe_encode_bare_jid(WithJID, Env), - remote_resource => jid_to_non_empty_resource(WithJID)}. - -spec index_hint_sql(env_vars()) -> string(). index_hint_sql(#{host := Host}) -> case mongoose_rdbms:db_engine(Host) of @@ -228,7 +209,7 @@ archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> end. do_archive_message(Host, Params, Env) -> - Row = prepare_message_with_env(Params, Env), + Row = mam_encoder:encode_message(Params, Env, db_mappings()), {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_message, Row). %% Retraction logic @@ -249,8 +230,8 @@ retract_message(Host, #{archive_id := UserID, remote_jid := RemJID, end. get_retraction_info(Host, UserID, RemJID, OriginID, Dir, Env) -> - BinBareRemJID = encode_jid(jid:to_bare(RemJID), Env), - BinDir = encode_direction(Dir), + BinBareRemJID = mam_encoder:encode_jid(jid:to_bare(RemJID), Env), + BinDir = mam_encoder:encode_direction(Dir), {selected, Rows} = execute_select_messages_to_retract( Host, UserID, BinBareRemJID, OriginID, BinDir), decode_retraction_info(Env, Rows). @@ -267,7 +248,7 @@ make_tombstone(_Host, UserID, OriginID, skip, _Env) -> user_id => UserID, origin_id => OriginID}); make_tombstone(Host, UserID, OriginID, #{packet := Packet, message_id := MessID}, Env) -> Tombstone = mod_mam_utils:tombstone(Packet, OriginID), - TombstoneData = encode_packet(Tombstone, Env), + TombstoneData = mam_encoder:encode_packet(Tombstone, Env), execute_make_tombstone(Host, TombstoneData, UserID, MessID). execute_select_messages_to_retract(Host, UserID, BareRemJID, OriginID, Dir) -> @@ -284,14 +265,7 @@ execute_make_tombstone(Host, TombstoneData, UserID, MessID) -> -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). prepare_message(Host, Params = #{local_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), - prepare_message_with_env(Params, Env). - -prepare_message_with_env(Params, Env) -> - [prepare_value(Params, Env, Mapping) || Mapping <- db_mappings()]. - -prepare_value(Params, Env, #db_mapping{param = Param, format = Format}) -> - Value = maps:get(Param, Params), - encode_value(Format, Value, Env). + mam_encoder:encode_message(Params, Env, db_mappings()). column_names(Mappings) -> [Column || #db_mapping{column = Column} <- Mappings]. @@ -328,7 +302,7 @@ lookup_messages({error, _Reason}=Result, _Host, _Params) -> Result; lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), - ExtParams = extend_lookup_params(Params, Env), + ExtParams = mam_encoder:extend_lookup_params(Params, Env), Filter = mam_filter:produce_filter(ExtParams, lookup_fields()), mam_lookup:lookup(Env, Filter, ExtParams). @@ -344,61 +318,10 @@ lookup_query(QueryType, Env, Filters, Order) -> %% ---------------------------------------------------------------------- %% Optimizations and extensible code --spec encode_value(value_type(), term(), env_vars()) -> term(). -encode_value(int, Value, _Env) when is_integer(Value) -> - Value; -encode_value(maybe_string, none, _Env) -> - null; -encode_value(maybe_string, Value, _Env) when is_binary(Value) -> - Value; -encode_value(direction, Value, _Env) -> - encode_direction(Value); -encode_value(bare_jid, Value, Env) -> - encode_jid(jid:to_bare(Value), Env); -encode_value(jid, Value, Env) -> - encode_jid(Value, Env); -encode_value(jid_resource, #jid{lresource = Res}, _Env) -> - Res; -encode_value(xml, Value, Env) -> - encode_packet(Value, Env); -encode_value(search, Value, Env) -> - encode_search_body(Value, Env). - -encode_direction(incoming) -> <<"I">>; -encode_direction(outgoing) -> <<"O">>. - -make_start_id(Start, Borders) -> - StartID = maybe_encode_compact_uuid(Start, 0), - apply_start_border(Borders, StartID). - -make_end_id(End, Borders) -> - EndID = maybe_encode_compact_uuid(End, 255), - apply_end_border(Borders, EndID). - -maybe_encode_compact_uuid(undefined, _) -> - undefined; -maybe_encode_compact_uuid(Microseconds, NodeID) -> - encode_compact_uuid(Microseconds, NodeID). - -jid_to_non_empty_resource(undefined) -> undefined; -jid_to_non_empty_resource(#jid{lresource = <<>>}) -> undefined; -jid_to_non_empty_resource(#jid{lresource = Res}) -> Res. - -maybe_encode_bare_jid(undefined, _Env) -> undefined; -maybe_encode_bare_jid(JID, Env) -> encode_jid(jid:to_bare(JID), Env). - --spec encode_jid(jid:jid(), env_vars()) -> binary(). -encode_jid(JID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> - mam_jid:encode(Codec, ArcJID, JID). - -spec decode_jid(binary(), env_vars()) -> jid:jid(). decode_jid(EncodedJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> mam_jid:decode(Codec, ArcJID, EncodedJID). --spec encode_packet(exml:element(), env_vars()) -> binary(). -encode_packet(Packet, #{db_message_codec := Codec}) -> - mam_message:encode(Codec, Packet). - -spec decode_packet(binary(), env_vars()) -> exml:element(). decode_packet(EncBin, Env = #{db_message_codec := Codec}) -> Bin = unescape_binary(EncBin, Env), @@ -413,10 +336,6 @@ unescape_binary(Bin, #{host := Host}) -> get_retract_id(Packet, #{has_message_retraction := Enabled}) -> mod_mam_utils:get_retract_id(Enabled, Packet). --spec encode_search_body(exml:element(), env_vars()) -> binary(). -encode_search_body(Packet, #{has_full_text_search := SearchEnabled}) -> - mod_mam_utils:packet_to_search_body(SearchEnabled, Packet). - -spec db_jid_codec(jid:server(), module()) -> module(). db_jid_codec(Host, Module) -> gen_mod:get_module_opt(Host, Module, db_jid_format, mam_jid_mini). From 9faf15488299e0c4656d7b0baa589af53afa3094 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 18:54:16 +0100 Subject: [PATCH 33/92] Use Ext prefix in decoders --- src/mam/mod_mam_rdbms_arch.erl | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 729a81f5b8..f8022f5dd3 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -230,10 +230,13 @@ retract_message(Host, #{archive_id := UserID, remote_jid := RemJID, end. get_retraction_info(Host, UserID, RemJID, OriginID, Dir, Env) -> - BinBareRemJID = mam_encoder:encode_jid(jid:to_bare(RemJID), Env), - BinDir = mam_encoder:encode_direction(Dir), + %% Code style notice: + %% - Add Ext prefix for all externally encoded data + %% (in cases, when we usually add Bin, B, S Esc prefixes) + ExtBareRemJID = mam_encoder:encode_jid(jid:to_bare(RemJID), Env), + ExtDir = mam_encoder:encode_direction(Dir), {selected, Rows} = execute_select_messages_to_retract( - Host, UserID, BinBareRemJID, OriginID, BinDir), + Host, UserID, ExtBareRemJID, OriginID, ExtDir), decode_retraction_info(Env, Rows). decode_retraction_info(_Env, []) -> skip; @@ -302,14 +305,14 @@ lookup_messages({error, _Reason}=Result, _Host, _Params) -> Result; lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), - ExtParams = mam_encoder:extend_lookup_params(Params, Env), - Filter = mam_filter:produce_filter(ExtParams, lookup_fields()), + ExdParams = mam_encoder:extend_lookup_params(Params, Env), + Filter = mam_filter:produce_filter(ExdParams, lookup_fields()), mam_lookup:lookup(Env, Filter, ExtParams). -row_to_uniform_format({BMessID, BSrcJID, Data}, Env) -> - MessID = mongoose_rdbms:result_to_integer(BMessID), - SrcJID = decode_jid(BSrcJID, Env), - Packet = decode_packet(Data, Env), +row_to_uniform_format({ExtMessID, ExtSrcJID, ExtData}, Env) -> + MessID = mongoose_rdbms:result_to_integer(ExtMessID), + SrcJID = decode_jid(ExtSrcJID, Env), + Packet = decode_packet(ExtData, Env), {MessID, SrcJID, Packet}. lookup_query(QueryType, Env, Filters, Order) -> @@ -319,12 +322,12 @@ lookup_query(QueryType, Env, Filters, Order) -> %% Optimizations and extensible code -spec decode_jid(binary(), env_vars()) -> jid:jid(). -decode_jid(EncodedJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> - mam_jid:decode(Codec, ArcJID, EncodedJID). +decode_jid(ExtJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> + mam_jid:decode(Codec, ArcJID, ExtJID). -spec decode_packet(binary(), env_vars()) -> exml:element(). -decode_packet(EncBin, Env = #{db_message_codec := Codec}) -> - Bin = unescape_binary(EncBin, Env), +decode_packet(ExtBin, Env = #{db_message_codec := Codec}) -> + Bin = unescape_binary(ExtBin, Env), mam_message:decode(Codec, Bin). -spec unescape_binary(binary(), env_vars()) -> binary(). From 4c16d74bfde15988d01da184cc9687fe9ad7fafa Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 19:01:23 +0100 Subject: [PATCH 34/92] Move common code into mam_decoder --- src/mam/mam_decoder.erl | 31 +++++++++++++++++++++++++++++++ src/mam/mod_mam_rdbms_arch.erl | 33 +++++---------------------------- 2 files changed, 36 insertions(+), 28 deletions(-) create mode 100644 src/mam/mam_decoder.erl diff --git a/src/mam/mam_decoder.erl b/src/mam/mam_decoder.erl new file mode 100644 index 0000000000..694ed4e894 --- /dev/null +++ b/src/mam/mam_decoder.erl @@ -0,0 +1,31 @@ +-module(mam_decoder). +-export([decode_row/2]). +-export([decode_retraction_info/2]). + +-type env_vars() :: map(). + +decode_row({ExtMessID, ExtSrcJID, ExtData}, Env) -> + MessID = mongoose_rdbms:result_to_integer(ExtMessID), + SrcJID = decode_jid(ExtSrcJID, Env), + Packet = decode_packet(ExtData, Env), + {MessID, SrcJID, Packet}. + +decode_retraction_info(_Env, []) -> skip; +decode_retraction_info(Env, [{ResMessID, Data}]) -> + Packet = decode_packet(Data, Env), + MessID = mongoose_rdbms:result_to_integer(ResMessID), + #{packet => Packet, message_id => MessID}. + +-spec decode_jid(binary(), env_vars()) -> jid:jid(). +decode_jid(ExtJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> + mam_jid:decode(Codec, ArcJID, ExtJID). + +-spec decode_packet(binary(), env_vars()) -> exml:element(). +decode_packet(ExtBin, Env = #{db_message_codec := Codec}) -> + Bin = unescape_binary(ExtBin, Env), + mam_message:decode(Codec, Bin). + +-spec unescape_binary(binary(), env_vars()) -> binary(). +unescape_binary(Bin, #{host := Host}) -> + %% Funny, rdbms ignores this Host variable + mongoose_rdbms:unescape_binary(Host, Bin). diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index f8022f5dd3..3cb68157e0 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -91,6 +91,9 @@ env_vars(Host, ArcJID) -> db_jid_codec => db_jid_codec(Host, ?MODULE), db_message_codec => db_message_codec(Host, ?MODULE)}. +row_to_uniform_format(Row, Env) -> + mam_decoder:decode_row(Row, Env). + -spec index_hint_sql(env_vars()) -> string(). index_hint_sql(#{host := Host}) -> case mongoose_rdbms:db_engine(Host) of @@ -237,13 +240,7 @@ get_retraction_info(Host, UserID, RemJID, OriginID, Dir, Env) -> ExtDir = mam_encoder:encode_direction(Dir), {selected, Rows} = execute_select_messages_to_retract( Host, UserID, ExtBareRemJID, OriginID, ExtDir), - decode_retraction_info(Env, Rows). - -decode_retraction_info(_Env, []) -> skip; -decode_retraction_info(Env, [{ResMessID, Data}]) -> - Packet = decode_packet(Data, Env), - MessID = mongoose_rdbms:result_to_integer(ResMessID), - #{packet => Packet, message_id => MessID}. + mam_decoder:decode_retraction_info(Env, Rows). make_tombstone(_Host, UserID, OriginID, skip, _Env) -> ?LOG_INFO(#{what => make_tombstone_failed, @@ -307,13 +304,7 @@ lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), ExdParams = mam_encoder:extend_lookup_params(Params, Env), Filter = mam_filter:produce_filter(ExdParams, lookup_fields()), - mam_lookup:lookup(Env, Filter, ExtParams). - -row_to_uniform_format({ExtMessID, ExtSrcJID, ExtData}, Env) -> - MessID = mongoose_rdbms:result_to_integer(ExtMessID), - SrcJID = decode_jid(ExtSrcJID, Env), - Packet = decode_packet(ExtData, Env), - {MessID, SrcJID, Packet}. + mam_lookup:lookup(Env, Filter, ExdParams). lookup_query(QueryType, Env, Filters, Order) -> mam_lookup_sql:lookup_query(QueryType, Env, Filters, Order). @@ -321,20 +312,6 @@ lookup_query(QueryType, Env, Filters, Order) -> %% ---------------------------------------------------------------------- %% Optimizations and extensible code --spec decode_jid(binary(), env_vars()) -> jid:jid(). -decode_jid(ExtJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) -> - mam_jid:decode(Codec, ArcJID, ExtJID). - --spec decode_packet(binary(), env_vars()) -> exml:element(). -decode_packet(ExtBin, Env = #{db_message_codec := Codec}) -> - Bin = unescape_binary(ExtBin, Env), - mam_message:decode(Codec, Bin). - --spec unescape_binary(binary(), env_vars()) -> binary(). -unescape_binary(Bin, #{host := Host}) -> - %% Funny, rdbms ignores this Host variable - mongoose_rdbms:unescape_binary(Host, Bin). - -spec get_retract_id(exml:element(), env_vars()) -> none | binary(). get_retract_id(Packet, #{has_message_retraction := Enabled}) -> mod_mam_utils:get_retract_id(Enabled, Packet). From 17e22c74b22a97ca44232c465253e269aafc3704 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 19:11:09 +0100 Subject: [PATCH 35/92] Reorder code in mod_mam_rdbms_arch --- src/mam/mod_mam_rdbms_arch.erl | 185 +++++++++++++++++---------------- 1 file changed, 96 insertions(+), 89 deletions(-) diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 3cb68157e0..b369474d63 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -56,6 +56,89 @@ -type env_vars() :: map(). +%% ---------------------------------------------------------------------- +%% gen_mod callbacks +%% Starting and stopping functions for users' archives + +-spec start(jid:server(), _) -> ok. +start(Host, Opts) -> + start_pm(Host, Opts), + register_prepared_queries(), + ok. + +-spec stop(jid:server()) -> ok. +stop(Host) -> + stop_pm(Host). + +-spec get_mam_pm_gdpr_data(ejabberd_gen_mam_archive:mam_pm_gdpr_data(), jid:jid()) -> + ejabberd_gen_mam_archive:mam_pm_gdpr_data(). +get_mam_pm_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> + case mod_mam:archive_id(Host, User) of + undefined -> []; + ArchiveID -> + Env = env_vars(Host, ArcJID), + {selected, Rows} = extract_gdpr_messages(Env, ArchiveID), + [uniform_to_gdpr(row_to_uniform_format(Row, Env)) || Row <- Rows] ++ Acc + end. + +uniform_to_gdpr({MessID, RemoteJID, Packet}) -> + {integer_to_binary(MessID), jid:to_binary(RemoteJID), exml:to_binary(Packet)}. + +%% ---------------------------------------------------------------------- +%% Add hooks for mod_mam + +-spec start_pm(jid:server(), _) -> 'ok'. +start_pm(Host, _Opts) -> + case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of + true -> + ok; + false -> + ejabberd_hooks:add(mam_archive_message, Host, ?MODULE, archive_message, 50) + end, + ejabberd_hooks:add(mam_archive_size, Host, ?MODULE, archive_size, 50), + ejabberd_hooks:add(mam_lookup_messages, Host, ?MODULE, lookup_messages, 50), + ejabberd_hooks:add(mam_remove_archive, Host, ?MODULE, remove_archive, 50), + ejabberd_hooks:add(get_mam_pm_gdpr_data, Host, ?MODULE, get_mam_pm_gdpr_data, 50), + ok. + + +-spec stop_pm(jid:server()) -> ok. +stop_pm(Host) -> + case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of + true -> + ok; + false -> + ejabberd_hooks:delete(mam_archive_message, Host, ?MODULE, archive_message, 50) + end, + ejabberd_hooks:delete(mam_archive_size, Host, ?MODULE, archive_size, 50), + ejabberd_hooks:delete(mam_lookup_messages, Host, ?MODULE, lookup_messages, 50), + ejabberd_hooks:delete(mam_remove_archive, Host, ?MODULE, remove_archive, 50), + ejabberd_hooks:delete(get_mam_pm_gdpr_data, Host, ?MODULE, get_mam_pm_gdpr_data, 50), + ok. + +%% ---------------------------------------------------------------------- +%% SQL queries + +register_prepared_queries() -> + prepare_insert(insert_mam_message, 1), + mongoose_rdbms:prepare(mam_archive_remove, mam_message, [user_id], + [<<"DELETE FROM mam_message " + "WHERE user_id = ?">>]), + mongoose_rdbms:prepare(mam_make_tombstone, mam_message, [message, user_id, id], + [<<"UPDATE mam_message SET message = ?, search_body = '' " + "WHERE user_id = ? AND id = ?">>]), + {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(1), + mongoose_rdbms:prepare(mam_select_messages_to_retract, mam_message, + [user_id, remote_bare_jid, origin_id, direction], + [<<"SELECT ", LimitMSSQL/binary, + " id, message FROM mam_message" + " WHERE user_id = ? AND remote_bare_jid = ? " + " AND origin_id = ? AND direction = ?" + " ORDER BY id DESC ", LimitSQL/binary>>]). + +%% ---------------------------------------------------------------------- +%% Declarative logic + db_mappings() -> [#db_mapping{column = id, param = message_id, format = int}, #db_mapping{column = user_id, param = archive_id, format = int}, @@ -113,79 +196,23 @@ column_to_id(search_body) -> "s"; column_to_id(limit) -> "l"; column_to_id(offset) -> "o". -%% ---------------------------------------------------------------------- -%% gen_mod callbacks -%% Starting and stopping functions for users' archives - --spec start(jid:server(), _) -> ok. -start(Host, Opts) -> - start_pm(Host, Opts), - prepare_insert(insert_mam_message, 1), - mongoose_rdbms:prepare(mam_archive_remove, mam_message, [user_id], - [<<"DELETE FROM mam_message " - "WHERE user_id = ?">>]), - mongoose_rdbms:prepare(mam_make_tombstone, mam_message, [message, user_id, id], - [<<"UPDATE mam_message SET message = ?, search_body = '' " - "WHERE user_id = ? AND id = ?">>]), - {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(1), - mongoose_rdbms:prepare(mam_select_messages_to_retract, mam_message, - [user_id, remote_bare_jid, origin_id, direction], - [<<"SELECT ", LimitMSSQL/binary, - " id, message FROM mam_message" - " WHERE user_id = ? AND remote_bare_jid = ? " - " AND origin_id = ? AND direction = ?" - " ORDER BY id DESC ", LimitSQL/binary>>]), - ok. - --spec stop(jid:server()) -> ok. -stop(Host) -> - stop_pm(Host). - --spec get_mam_pm_gdpr_data(ejabberd_gen_mam_archive:mam_pm_gdpr_data(), jid:jid()) -> - ejabberd_gen_mam_archive:mam_pm_gdpr_data(). -get_mam_pm_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> - case mod_mam:archive_id(Host, User) of - undefined -> []; - ArchiveID -> - Env = env_vars(Host, ArcJID), - {selected, Rows} = extract_gdpr_messages(Env, ArchiveID), - [uniform_to_gdpr(row_to_uniform_format(Row, Env)) || Row <- Rows] ++ Acc - end. - -uniform_to_gdpr({MessID, RemoteJID, Packet}) -> - {integer_to_binary(MessID), jid:to_binary(RemoteJID), exml:to_binary(Packet)}. +column_names(Mappings) -> + [Column || #db_mapping{column = Column} <- Mappings]. %% ---------------------------------------------------------------------- -%% Add hooks for mod_mam +%% Options --spec start_pm(jid:server(), _) -> 'ok'. -start_pm(Host, _Opts) -> - case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of - true -> - ok; - false -> - ejabberd_hooks:add(mam_archive_message, Host, ?MODULE, archive_message, 50) - end, - ejabberd_hooks:add(mam_archive_size, Host, ?MODULE, archive_size, 50), - ejabberd_hooks:add(mam_lookup_messages, Host, ?MODULE, lookup_messages, 50), - ejabberd_hooks:add(mam_remove_archive, Host, ?MODULE, remove_archive, 50), - ejabberd_hooks:add(get_mam_pm_gdpr_data, Host, ?MODULE, get_mam_pm_gdpr_data, 50), - ok. +-spec db_jid_codec(jid:server(), module()) -> module(). +db_jid_codec(Host, Module) -> + gen_mod:get_module_opt(Host, Module, db_jid_format, mam_jid_mini). +-spec db_message_codec(jid:server(), module()) -> module(). +db_message_codec(Host, Module) -> + gen_mod:get_module_opt(Host, Module, db_message_format, mam_message_compressed_eterm). --spec stop_pm(jid:server()) -> ok. -stop_pm(Host) -> - case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of - true -> - ok; - false -> - ejabberd_hooks:delete(mam_archive_message, Host, ?MODULE, archive_message, 50) - end, - ejabberd_hooks:delete(mam_archive_size, Host, ?MODULE, archive_size, 50), - ejabberd_hooks:delete(mam_lookup_messages, Host, ?MODULE, lookup_messages, 50), - ejabberd_hooks:delete(mam_remove_archive, Host, ?MODULE, remove_archive, 50), - ejabberd_hooks:delete(get_mam_pm_gdpr_data, Host, ?MODULE, get_mam_pm_gdpr_data, 50), - ok. +-spec get_retract_id(exml:element(), env_vars()) -> none | binary(). +get_retract_id(Packet, #{has_message_retraction := Enabled}) -> + mod_mam_utils:get_retract_id(Enabled, Packet). %% ---------------------------------------------------------------------- %% Internal functions and callbacks @@ -208,7 +235,7 @@ archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> ?LOG_ERROR(#{what => archive_message_failed, host => Host, mam_params => Params, class => Class, reason => Reason, stacktrace => StackTrace}), - {error, Reason} + erlang:raise(Class, Reason, StackTrace) end. do_archive_message(Host, Params, Env) -> @@ -261,15 +288,11 @@ execute_make_tombstone(Host, TombstoneData, UserID, MessID) -> [TombstoneData, UserID, MessID]). %% Insert logic - -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). prepare_message(Host, Params = #{local_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), mam_encoder:encode_message(Params, Env, db_mappings()). -column_names(Mappings) -> - [Column || #db_mapping{column = Column} <- Mappings]. - -spec prepare_insert(Name :: atom(), NumRows :: pos_integer()) -> ok. prepare_insert(Name, NumRows) -> Table = mam_message, @@ -278,7 +301,6 @@ prepare_insert(Name, NumRows) -> mongoose_rdbms:prepare(Name, Table, Fields2, Query), ok. - %% Removal logic -spec remove_archive(Acc :: mongoose_acc:t(), Host :: jid:server(), ArchiveID :: mod_mam:archive_id(), @@ -308,18 +330,3 @@ lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> lookup_query(QueryType, Env, Filters, Order) -> mam_lookup_sql:lookup_query(QueryType, Env, Filters, Order). - -%% ---------------------------------------------------------------------- -%% Optimizations and extensible code - --spec get_retract_id(exml:element(), env_vars()) -> none | binary(). -get_retract_id(Packet, #{has_message_retraction := Enabled}) -> - mod_mam_utils:get_retract_id(Enabled, Packet). - --spec db_jid_codec(jid:server(), module()) -> module(). -db_jid_codec(Host, Module) -> - gen_mod:get_module_opt(Host, Module, db_jid_format, mam_jid_mini). - --spec db_message_codec(jid:server(), module()) -> module(). -db_message_codec(Host, Module) -> - gen_mod:get_module_opt(Host, Module, db_message_format, mam_message_compressed_eterm). From 479113d0c039a405ed6c2258260b8fec66c6f6a1 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 21:17:54 +0100 Subject: [PATCH 36/92] Use generic logic for mod_mam_muc_rdbms_arch --- src/mam/mam_decoder.erl | 7 + src/mam/mam_filter.erl | 15 + src/mam/mod_mam_muc_rdbms_arch.erl | 755 +++++------------- .../mod_mam_muc_rdbms_async_pool_writer.erl | 3 +- src/mam/mod_mam_rdbms_arch.erl | 63 +- 5 files changed, 267 insertions(+), 576 deletions(-) diff --git a/src/mam/mam_decoder.erl b/src/mam/mam_decoder.erl index 694ed4e894..3647b002ae 100644 --- a/src/mam/mam_decoder.erl +++ b/src/mam/mam_decoder.erl @@ -1,5 +1,6 @@ -module(mam_decoder). -export([decode_row/2]). +-export([decode_muc_row/2]). -export([decode_retraction_info/2]). -type env_vars() :: map(). @@ -10,6 +11,12 @@ decode_row({ExtMessID, ExtSrcJID, ExtData}, Env) -> Packet = decode_packet(ExtData, Env), {MessID, SrcJID, Packet}. +decode_muc_row({ExtMessID, Nick, ExtData}, Env = #{archive_jid := RoomJID}) -> + MessID = mongoose_rdbms:result_to_integer(ExtMessID), + SrcJID = jid:replace_resource(RoomJID, Nick), + Packet = decode_packet(ExtData, Env), + {MessID, SrcJID, Packet}. + decode_retraction_info(_Env, []) -> skip; decode_retraction_info(Env, [{ResMessID, Data}]) -> Packet = decode_packet(Data, Env), diff --git a/src/mam/mam_filter.erl b/src/mam/mam_filter.erl index 40ca9a0891..929daf3fa3 100644 --- a/src/mam/mam_filter.erl +++ b/src/mam/mam_filter.erl @@ -3,6 +3,21 @@ -export([produce_filter/2]). -include("mongoose_mam.hrl"). +-type column() :: atom(). + +-type filter_field() :: {like, column(), binary()} + | {le, column(), integer()} + | {ge, column(), integer()} + | {equal, column(), integer() | binary()} + | {lower, column(), integer()} + | {greater, column(), integer()} + | {limit, limit, integer()} + | {offset, offset, integer()}. +-type filter() :: [filter_field()]. + +-export_type([filter_field/0]). +-export_type([filter/0]). + -define(SEARCH_WORDS_LIMIT, 10). produce_filter(Params, Fields) -> diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 6e1537bd10..b28f968f15 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -1,7 +1,7 @@ %%%------------------------------------------------------------------- %%% @author Uvarov Michael %%% @copyright (C) 2013, Uvarov Michael -%%% @doc A backend for storing messages from MUC rooms using RDBMS. +%%% @doc RDBMS backend for MUC Message Archive Management. %%% @end %%%------------------------------------------------------------------- -module(mod_mam_muc_rdbms_arch). @@ -14,6 +14,8 @@ %% MAM hook handlers -behaviour(ejabberd_gen_mam_archive). +-behaviour(gen_mod). +-behaviour(mongoose_module_metrics). -callback encode(term()) -> binary(). -callback decode(binary()) -> term(). @@ -23,75 +25,60 @@ lookup_messages/3, remove_archive/4]). +-export([get_mam_muc_gdpr_data/2]). + %% Called from mod_mam_rdbms_async_writer -export([prepare_message/2, retract_message/2, prepare_insert/2]). - -%gdpr --export([get_mam_muc_gdpr_data/2]). +-export([extend_params_with_sender_id/2]). %% ---------------------------------------------------------------------- %% Imports -%% UMessID --import(mod_mam_utils, - [encode_compact_uuid/2]). - -%% Other --import(mod_mam_utils, - [apply_start_border/2, - apply_end_border/2]). - --import(mongoose_rdbms, - [escape_integer/1, - use_escaped_string/1, - use_escaped_integer/1]). - -include("mongoose.hrl"). -include("jlib.hrl"). -include_lib("exml/include/exml.hrl"). -include("mongoose_rsm.hrl"). - +-include("mongoose_mam.hrl"). %% ---------------------------------------------------------------------- %% Types --type filter() :: iolist(). --type escaped_message_id() :: mongoose_rdbms:escaped_integer(). --type escaped_room_id() :: mongoose_rdbms:escaped_integer(). --type escaped_jid() :: mongoose_rdbms:escaped_string(). --type unix_timestamp() :: mod_mam:unix_timestamp(). --type packet() :: any(). --type raw_row() :: {binary(), binary(), binary()}. +-type column() :: atom(). +-type env_vars() :: map(). %% ---------------------------------------------------------------------- %% gen_mod callbacks %% Starting and stopping functions for users' archives --spec start(jid:server(), _) -> 'ok'. +-spec start(jid:server(), _) -> ok. start(Host, Opts) -> - prepare_insert(insert_mam_muc_message, 1), + start_hooks(Host, Opts), + register_prepared_queries(), + ok. - mongoose_rdbms:prepare(mam_muc_archive_size, mam_muc_message, [user_id], - [<<"SELECT COUNT(*) FROM mam_muc_message " - "WHERE room_id = ?">>]), +-spec stop(jid:server()) -> ok. +stop(Host) -> + stop_hooks(Host). - mongoose_rdbms:prepare(mam_muc_extract_gdpr_messages, mam_muc_message, - [sender_id], - [<<"SELECT id, message " - " FROM mam_muc_message " - " WHERE sender_id=? " - " ORDER BY id">>]), - start_muc(Host, Opts). +-spec get_mam_muc_gdpr_data(ejabberd_gen_mam_archive:mam_pm_gdpr_data(), jid:jid()) -> + ejabberd_gen_mam_archive:mam_pm_gdpr_data(). +get_mam_muc_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> + case mod_mam:archive_id(Host, User) of + undefined -> []; + ArcID -> + Env = env_vars(Host, ArcJID), + {selected, Rows} = extract_gdpr_messages(Env, ArcID), + [uniform_to_gdpr(row_to_uniform_format(Row, Env)) || Row <- Rows] ++ Acc + end. --spec stop(jid:server()) -> 'ok'. -stop(Host) -> - stop_muc(Host). +uniform_to_gdpr({MessID, _Nick, Packet}) -> + {integer_to_binary(MessID), exml:to_binary(Packet)}. %% ---------------------------------------------------------------------- -%% Add hooks for mod_mam_muc +%% Add hooks for mod_mam --spec start_muc(jid:server(), _) -> 'ok'. -start_muc(Host, _Opts) -> +-spec start_hooks(jid:server(), _) -> ok. +start_hooks(Host, _Opts) -> case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of true -> ok; @@ -105,8 +92,8 @@ start_muc(Host, _Opts) -> ok. --spec stop_muc(jid:server()) -> 'ok'. -stop_muc(Host) -> +-spec stop_hooks(jid:server()) -> ok. +stop_hooks(Host) -> case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of true -> ok; @@ -119,531 +106,211 @@ stop_muc(Host) -> ejabberd_hooks:delete(get_mam_muc_gdpr_data, Host, ?MODULE, get_mam_muc_gdpr_data, 50), ok. +%% ---------------------------------------------------------------------- +%% SQL queries + +register_prepared_queries() -> + prepare_insert(insert_mam_muc_message, 1), + mongoose_rdbms:prepare(mam_muc_archive_remove, mam_muc_message, [room_id], + [<<"DELETE FROM mam_muc_message " + "WHERE room_id = ?">>]), + mongoose_rdbms:prepare(mam_muc_make_tombstone, mam_muc_message, [message, room_id, id], + [<<"UPDATE mam_muc_message SET message = ?, search_body = '' " + "WHERE room_id = ? AND id = ?">>]), + {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(1), + mongoose_rdbms:prepare(mam_muc_select_messages_to_retract, mam_muc_message, + [room_id, sender_id, origin_id], + [<<"SELECT ", LimitMSSQL/binary, + " id, message FROM mam_muc_message" + " WHERE room_id = ? AND sender_id = ? " + " AND origin_id = ?" + " ORDER BY id DESC ", LimitSQL/binary>>]). + +%% ---------------------------------------------------------------------- +%% Declarative logic + +db_mappings() -> + [#db_mapping{column = id, param = message_id, format = int}, + #db_mapping{column = room_id, param = archive_id, format = int}, + #db_mapping{column = sender_id, param = sender_id, format = int}, + #db_mapping{column = nick_name, param = source_jid, format = jid_resource}, + #db_mapping{column = origin_id, param = origin_id, format = maybe_string}, + #db_mapping{column = message, param = packet, format = xml}, + #db_mapping{column = search_body, param = packet, format = search}]. + +lookup_fields() -> + [#lookup_field{op = equal, column = room_id, param = archive_id, required = true}, + #lookup_field{op = ge, column = id, param = start_id}, + #lookup_field{op = le, column = id, param = end_id}, + #lookup_field{op = equal, column = nick_name, param = remote_resource}, + #lookup_field{op = like, column = search_body, param = norm_search_text, value_maker = search_words}]. + +env_vars(Host, ArcJID) -> + %% Please, minimize the usage of the host field. + %% It's only for passing into RDBMS. + #{host => Host, + archive_jid => ArcJID, + table => mam_muc_message, + index_hint_fn => fun index_hint_sql/1, + columns_sql_fn => fun columns_sql/1, + column_to_id_fn => fun column_to_id/1, + lookup_fn => fun lookup_query/4, + decode_row_fn => fun row_to_uniform_format/2, + has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam_muc, Host), + has_full_text_search => mod_mam_utils:has_full_text_search(mod_mam_muc, Host), + db_jid_codec => db_jid_codec(Host, ?MODULE), + db_message_codec => db_message_codec(Host, ?MODULE)}. + +row_to_uniform_format(Row, Env) -> + mam_decoder:decode_muc_row(Row, Env). + +-spec index_hint_sql(env_vars()) -> string(). +index_hint_sql(_) -> "". + +columns_sql(lookup) -> "id, nick_name, message"; +columns_sql(count) -> "COUNT(*)". + +column_to_id(id) -> "i"; +column_to_id(room_id) -> "u"; +column_to_id(nick_name) -> "n"; +column_to_id(search_body) -> "s"; +%% fictional columns +column_to_id(limit) -> "l"; +column_to_id(offset) -> "o". + +column_names(Mappings) -> + [Column || #db_mapping{column = Column} <- Mappings]. + +%% ---------------------------------------------------------------------- +%% Options + +-spec db_jid_codec(jid:server(), module()) -> module(). +db_jid_codec(Host, Module) -> + gen_mod:get_module_opt(Host, Module, db_jid_format, mam_jid_rfc). + +-spec db_message_codec(jid:server(), module()) -> module(). +db_message_codec(Host, Module) -> + gen_mod:get_module_opt(Host, Module, db_message_format, mam_message_compressed_eterm). + +-spec get_retract_id(exml:element(), env_vars()) -> none | binary(). +get_retract_id(Packet, #{has_message_retraction := Enabled}) -> + mod_mam_utils:get_retract_id(Enabled, Packet). %% ---------------------------------------------------------------------- %% Internal functions and callbacks --spec archive_size(integer(), jid:server(), integer(), jid:jid()) - -> integer(). -archive_size(Size, Host, RoomID, _RoomJID) when is_integer(Size) -> - Result = mod_mam_utils:success_sql_execute(Host, mam_muc_archive_size, [RoomID]), +-spec archive_size(Size :: integer(), Host :: jid:server(), + ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). +archive_size(Size, Host, ArcID, ArcJID) when is_integer(Size) -> + Filter = [{equal, room_id, ArcID}], + Env = env_vars(Host, ArcJID), + Result = lookup_query(count, Env, Filter, unordered), mongoose_rdbms:selected_to_integer(Result). +extend_params_with_sender_id(Host, Params = #{remote_jid := SenderJID}) -> + BareSenderJID = jid:to_bare(SenderJID), + SenderID = mod_mam:archive_id_int(Host, BareSenderJID), + Params#{sender_id => SenderID}. + -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok. -archive_message(_Result, Host, Params = #{direction := incoming}) -> +archive_message(_Result, Host, Params0 = #{local_jid := ArcJID}) -> try - Row = prepare_message(Host, Params), - {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_muc_message, Row), - retract_message(Host, Params), + Params = extend_params_with_sender_id(Host, Params0), + Env = env_vars(Host, ArcJID), + do_archive_message(Host, Params, Env), + retract_message(Host, Params, Env), ok - catch _Type:Reason:StackTrace -> - ?LOG_ERROR(#{what => archive_message_failed, - host => Host, mam_params => Params, - reason => Reason, stacktrace => StackTrace}), - {error, Reason} + catch Class:Reason:StackTrace -> + ?LOG_ERROR(#{what => archive_message_failed, + host => Host, mam_params => Params0, + class => Class, reason => Reason, stacktrace => StackTrace}), + erlang:raise(Class, Reason, StackTrace) end. -retract_message(Host, #{archive_id := RoomID, - remote_jid := SenderJID, - packet := Packet}) -> - case mod_mam_utils:get_retract_id(mod_mam_muc, Host, Packet) of +do_archive_message(Host, Params, Env) -> + Row = mam_encoder:encode_message(Params, Env, db_mappings()), + {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_muc_message, Row). + +%% Retraction logic +%% Called after inserting a new message +-spec retract_message(jid:server(), mod_mam:archive_message_params()) -> ok. +retract_message(Host, #{local_jid := ArcJID} = Params) -> + Env = env_vars(Host, ArcJID), + retract_message(Host, Params, Env). + +-spec retract_message(jid:server(), mod_mam:archive_message_params(), env_vars()) -> ok. +retract_message(Host, #{archive_id := ArcID, sender_id := SenderID, + packet := Packet}, Env) -> + case get_retract_id(Packet, Env) of none -> ok; - OriginIDToRetract -> retract_message(Host, RoomID, SenderJID, OriginIDToRetract) + OriginID -> + Info = get_retraction_info(Host, ArcID, SenderID, OriginID, Env), + make_tombstone(Host, ArcID, OriginID, Info, Env) end. -retract_message(Host, RoomID, SenderJID, OriginID) -> - SRoomID = use_escaped_integer(escape_room_id(RoomID)), - SenderID = mod_mam:archive_id_int(Host, jid:to_bare(SenderJID)), - SSenderID = use_escaped_integer(mongoose_rdbms:escape_integer(SenderID)), - SOriginID = use_escaped_string(mongoose_rdbms:escape_string(OriginID)), - Query = query_for_messages_to_retract(SRoomID, SSenderID, SOriginID), - {selected, Rows} = mod_mam_utils:success_sql_query(Host, Query), - make_tombstone(Host, SRoomID, OriginID, Rows), - ok. +get_retraction_info(Host, ArcID, SenderID, OriginID, Env) -> + {selected, Rows} = + execute_select_messages_to_retract(Host, ArcID, SenderID, OriginID), + mam_decoder:decode_retraction_info(Env, Rows). -make_tombstone(_Host, SRoomID, OriginID, []) -> +make_tombstone(_Host, ArcID, OriginID, skip, _Env) -> ?LOG_INFO(#{what => make_tombstone_failed, text => <<"Message to retract was not found by origin id">>, - room_id => SRoomID, origin_id => OriginID}); -make_tombstone(Host, SRoomID, OriginID, [{ResMessID, ResData}]) -> - Data = mongoose_rdbms:unescape_binary(Host, ResData), - Packet = stored_binary_to_packet(Host, Data), - MessID = mongoose_rdbms:result_to_integer(ResMessID), + user_id => ArcID, origin_id => OriginID}); +make_tombstone(Host, ArcID, OriginID, #{packet := Packet, message_id := MessID}, Env) -> Tombstone = mod_mam_utils:tombstone(Packet, OriginID), - TombstoneData = packet_to_stored_binary(Host, Tombstone), - STombstoneData = mongoose_rdbms:use_escaped_binary( - mongoose_rdbms:escape_binary(Host, TombstoneData)), - BMessID = use_escaped_integer(escape_message_id(MessID)), - UpdateQuery = query_to_make_tombstone(STombstoneData, SRoomID, BMessID), - {updated, 1} = mod_mam_utils:success_sql_query(Host, UpdateQuery). - -query_for_messages_to_retract(SRoomID, SSenderID, SOriginID) -> - {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits(1), - ["SELECT ", LimitMSSQL, " id, message FROM mam_muc_message" - " WHERE room_id = ", SRoomID, " AND sender_id = ", SSenderID, " AND origin_id = ", SOriginID, - " ORDER BY id DESC ", LimitSQL]. - -query_to_make_tombstone(STombstoneData, SRoomID, BMessID) -> - ["UPDATE mam_muc_message SET message = ", STombstoneData, ", search_body = ''" - " WHERE room_id = ", SRoomID, " AND id = '", BMessID, "'"]. - --spec prepare_message(Host :: jid:server(), Params :: mod_mam:archive_message_params()) -> - [binary() | integer()]. -prepare_message(Host, #{message_id := MessID, - archive_id := RoomID, - remote_jid := SenderJID, - source_jid := #jid{lresource = FromNick}, - origin_id := OriginID, - packet := Packet}) -> - BareSenderJID = jid:to_bare(SenderJID), - Data = packet_to_stored_binary(Host, Packet), - TextBody = mod_mam_utils:packet_to_search_body(mod_mam_muc, Host, Packet), - SenderID = mod_mam:archive_id_int(Host, BareSenderJID), - SOriginID = case OriginID of - none -> null; - _ -> OriginID - end, - [MessID, RoomID, SenderID, FromNick, SOriginID, Data, TextBody]. + TombstoneData = mam_encoder:encode_packet(Tombstone, Env), + execute_make_tombstone(Host, TombstoneData, ArcID, MessID). + +execute_select_messages_to_retract(Host, ArcID, SenderID, OriginID) -> + mod_mam_utils:success_sql_execute(Host, mam_muc_select_messages_to_retract, + [ArcID, SenderID, OriginID]). + +execute_make_tombstone(Host, TombstoneData, ArcID, MessID) -> + {updated, _} = + mod_mam_utils:success_sql_execute(Host, mam_muc_make_tombstone, + [TombstoneData, ArcID, MessID]). + +%% Insert logic +-spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). +prepare_message(Host, Params = #{local_jid := ArcJID}) -> + Env = env_vars(Host, ArcJID), + mam_encoder:encode_message(Params, Env, db_mappings()). -spec prepare_insert(Name :: atom(), NumRows :: pos_integer()) -> ok. prepare_insert(Name, NumRows) -> Table = mam_muc_message, - Fields = [id, room_id, sender_id, nick_name, origin_id, message, search_body], + Fields = column_names(db_mappings()), {Query, Fields2} = rdbms_queries:create_bulk_insert_query(Table, Fields, NumRows), mongoose_rdbms:prepare(Name, Table, Fields2, Query), ok. - -lookup_messages({error, _Reason}=Result, _Host, _Params) -> - Result; -lookup_messages(_Result, Host, - #{archive_id := UserID, owner_jid := UserJID, rsm := RSM, - borders := Borders, start_ts := Start, end_ts := End, now := Now, - with_jid := WithJID, search_text := SearchText, page_size := PageSize, - is_simple := IsSimple}) -> - try - lookup_messages(Host, - UserID, UserJID, RSM, Borders, - Start, End, Now, WithJID, - mod_mam_utils:normalize_search_text(SearchText), - PageSize, IsSimple) - catch _Type:Reason:S -> - {error, {Reason, {stacktrace, S}}} - end. - --spec lookup_messages(Host :: jid:server(), - ArchiveID :: mod_mam:archive_id(), - ArchiveJID :: jid:jid(), - RSM :: jlib:rsm_in() | undefined, - Borders :: mod_mam:borders() | undefined, - Start :: mod_mam:unix_timestamp() | undefined, - End :: mod_mam:unix_timestamp() | undefined, - Now :: mod_mam:unix_timestamp(), - WithJID :: jid:jid() | undefined, - SearchText :: binary() | undefined, - PageSize :: integer(), - IsSimple :: boolean() | opt_count) -> - {ok, mod_mam:lookup_result()}. -lookup_messages(Host, RoomID, RoomJID = #jid{}, - #rsm_in{direction = aft, id = ID}, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, true) -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, after_id(ID, Filter), 0, PageSize, false), - {ok, {undefined, undefined, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - #rsm_in{direction = before, id = ID}, - Borders, Start, End, _Now, WithJID, SearchText, - PageSize, true) -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, before_id(ID, Filter), 0, PageSize, true), - {ok, {undefined, undefined, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - #rsm_in{direction = undefined, index = Offset}, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, true) -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), - {ok, {undefined, undefined, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - undefined, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, true) -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, Filter, 0, PageSize, false), - {ok, {undefined, undefined, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; -%% Cannot be optimized: -%% - #rsm_in{direction = aft, id = ID} -%% - #rsm_in{direction = before, id = ID} -lookup_messages(Host, RoomID, RoomJID = #jid{}, - #rsm_in{direction = before, id = undefined}, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, opt_count) -> - %% Last page - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, Filter, 0, PageSize, true), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {MessageRowsCount, 0, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; - false -> - FirstID = row_to_message_id(hd(MessageRows)), - Offset = calc_count(Host, before_id(FirstID, Filter)), - {ok, {Offset + MessageRowsCount, Offset, - rows_to_uniform_format(MessageRows, Host, RoomJID)}} - end; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - #rsm_in{direction = undefined, index = Offset}, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, opt_count) -> - %% By offset - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {Offset + MessageRowsCount, Offset, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; - false -> - LastID = row_to_message_id(lists:last(MessageRows)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), - {ok, {Offset + MessageRowsCount + CountAfterLastID, Offset, - rows_to_uniform_format(MessageRows, Host, RoomJID)}} - end; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - undefined, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, opt_count) -> - %% First page - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - MessageRows = extract_messages(Host, Filter, 0, PageSize, false), - MessageRowsCount = length(MessageRows), - case MessageRowsCount < PageSize of - true -> - {ok, {MessageRowsCount, 0, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}; - false -> - LastID = row_to_message_id(lists:last(MessageRows)), - CountAfterLastID = calc_count(Host, after_id(LastID, Filter)), - {ok, {MessageRowsCount + CountAfterLastID, 0, - rows_to_uniform_format(MessageRows, Host, RoomJID)}} - end; -lookup_messages(Host, RoomID, RoomJID = #jid{}, - RSM = #rsm_in{direction = aft, id = ID}, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, _) when ID =/= undefined -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - TotalCount = calc_count(Host, Filter), - Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, from_id(ID, Filter), 0, PageSize + 1, false), - Result = {TotalCount, Offset, rows_to_uniform_format(MessageRows, Host, RoomJID)}, - mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); -lookup_messages(Host, RoomID, RoomJID = #jid{}, - RSM = #rsm_in{direction = before, id = ID}, - Borders, Start, End, _Now, WithJID, SearchText, - PageSize, _) when ID =/= undefined -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - TotalCount = calc_count(Host, Filter), - Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, to_id(ID, Filter), 0, PageSize + 1, true), - Result = {TotalCount, Offset, rows_to_uniform_format(MessageRows, Host, RoomJID)}, - mod_mam_utils:check_for_item_not_found(RSM, PageSize, Result); -lookup_messages(Host, RoomID, RoomJID = #jid{}, - RSM, Borders, - Start, End, _Now, WithJID, SearchText, - PageSize, _) -> - Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText), - TotalCount = calc_count(Host, Filter), - Offset = calc_offset(Host, Filter, PageSize, TotalCount, RSM), - MessageRows = extract_messages(Host, Filter, Offset, PageSize, false), - {ok, {TotalCount, Offset, - rows_to_uniform_format(MessageRows, Host, RoomJID)}}. - --spec get_mam_muc_gdpr_data(ejabberd_gen_mam_archive:mam_muc_gdpr_data(), jid:jid()) -> - ejabberd_gen_mam_archive:mam_muc_gdpr_data(). -get_mam_muc_gdpr_data(Acc, #jid{ user = User, server = Host }) -> - case mod_mam:archive_id(Host, User) of - undefined -> Acc; - ArchiveID -> - {selected, Rows} = extract_gdpr_messages(Host, ArchiveID), - [{BMessID, gdpr_decode_packet(Host, SDataRaw)} || {BMessID, SDataRaw} <- Rows] ++ Acc - end. - --spec after_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). -after_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id > ", use_escaped_integer(SID)]. - --spec before_id(ID :: escaped_message_id() | undefined, - Filter :: filter()) -> filter(). -before_id(undefined, Filter) -> - Filter; -before_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id < ", use_escaped_integer(SID)]. - --spec from_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). -from_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id >= ", use_escaped_integer(SID)]. - --spec to_id(ID :: escaped_message_id(), Filter :: filter()) -> filter(). -to_id(ID, Filter) -> - SID = escape_message_id(ID), - [Filter, " AND id <= ", use_escaped_integer(SID)]. - - --spec rows_to_uniform_format([raw_row()], jid:server(), jid:jid()) -> - [mod_mam_muc:row()]. -rows_to_uniform_format(MessageRows, Host, RoomJID) -> - [do_row_to_uniform_format(Host, Row, RoomJID) || Row <- MessageRows]. - - --spec do_row_to_uniform_format(jid:server(), raw_row(), jid:jid()) -> - mod_mam_muc:row(). -do_row_to_uniform_format(Host, {BMessID, BNick, SDataRaw}, RoomJID) -> - MessID = mongoose_rdbms:result_to_integer(BMessID), - SrcJID = jid:replace_resource(RoomJID, BNick), - Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Packet = stored_binary_to_packet(Host, Data), - {MessID, SrcJID, Packet}. - - --spec row_to_message_id({binary(), _, _}) -> integer(). -row_to_message_id({BMessID, _, _}) -> - mongoose_rdbms:result_to_integer(BMessID). - - --spec remove_archive(map(), jid:server(), mod_mam:archive_id(), jid:jid()) -> map(). -remove_archive(Acc, Host, RoomID, _RoomJID) -> - {updated, _} = - mod_mam_utils:success_sql_query( - Host, - ["DELETE FROM mam_muc_message " - "WHERE room_id = ", use_escaped_integer(escape_room_id(RoomID))]), +%% Removal logic +-spec remove_archive(Acc :: mongoose_acc:t(), Host :: jid:server(), + ArcID :: mod_mam:archive_id(), + ArcJID :: jid:jid()) -> mongoose_acc:t(). +remove_archive(Acc, Host, ArcID, _ArcJID) -> + remove_archive(Host, ArcID), Acc. -%% @doc Columns are `["id", "nick_name", "message"]'. --spec extract_messages(Host :: jid:server(), - Filter :: filter(), IOffset :: non_neg_integer(), IMax :: pos_integer(), - ReverseLimit :: boolean()) -> [raw_row()]. -extract_messages(_Host, _Filter, _IOffset, 0, _) -> - []; -extract_messages(Host, Filter, IOffset, IMax, false) -> - {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, " ORDER BY id "), - ?LOG_DEBUG(#{what => mam_extract_messages, - text => <<"extract_messages query returns...">>, - mam_filter => Filter, offset => IOffset, max => IMax, - host => Host, message_rows => MessageRows}), - MessageRows; -extract_messages(Host, Filter, IOffset, IMax, true) -> - {selected, MessageRows} = - do_extract_messages(Host, Filter, IOffset, IMax, " ORDER BY id DESC "), - ?LOG_DEBUG(#{what => mam_extract_messages, - text => <<"extract_messages query returns...">>, - mam_filter => Filter, offset => IOffset, max => IMax, - host => Host, message_rows => MessageRows}), - lists:reverse(MessageRows). - -do_extract_messages(Host, Filter, 0, IMax, Order) -> - {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits(IMax), - mod_mam_utils:success_sql_query( - Host, - ["SELECT ", LimitMSSQL, " id, nick_name, message " - "FROM mam_muc_message ", - Filter, - Order, - " ", LimitSQL]); -do_extract_messages(Host, Filter, IOffset, IMax, Order) -> - {LimitSQL, _LimitMSSQL} = rdbms_queries:get_db_specific_limits(IMax), - Offset = rdbms_queries:get_db_specific_offset(IOffset, IMax), - mod_mam_utils:success_sql_query( - Host, - ["SELECT id, nick_name, message " - "FROM mam_muc_message ", - Filter, Order, LimitSQL, Offset]). - -extract_gdpr_messages(Host, ArchiveID) -> - mod_mam_utils:success_sql_execute(Host, mam_muc_extract_gdpr_messages, [ArchiveID]). - -%% @doc Zero-based index of the row with UMessID in the result test. -%% If the element does not exists, the MessID of the next element will -%% be returned instead. -%% -%% ``` -%% "SELECT COUNT(*) as "index" FROM mam_muc_message WHERE id <= '", UMessID -%% ''' --spec calc_index(Host :: jid:server(), - Filter :: iodata(), SUMessID :: escaped_message_id()) -> non_neg_integer(). -calc_index(Host, Filter, SUMessID) -> - {selected, [{BIndex}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) " - "FROM mam_muc_message ", - Filter, " AND id <= ", use_escaped_integer(SUMessID)]), - mongoose_rdbms:result_to_integer(BIndex). - - -%% @doc Count of elements in RSet before the passed element. -%% The element with the passed UMessID can be already deleted. -%% @end -%% "SELECT COUNT(*) as "count" FROM mam_muc_message WHERE id < '", UMessID --spec calc_before(Host :: jid:server(), - Filter :: iodata(), SUMessID :: escaped_message_id()) -> non_neg_integer(). -calc_before(Host, Filter, SUMessID) -> - {selected, [{BIndex}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) " - "FROM mam_muc_message ", - Filter, " AND id < ", use_escaped_integer(SUMessID)]), - mongoose_rdbms:result_to_integer(BIndex). - - -%% @doc Get the total result set size. -%% "SELECT COUNT(*) as "count" FROM mam_muc_message WHERE " --spec calc_count(Host :: jid:server(), - Filter :: filter()) -> non_neg_integer(). -calc_count(Host, Filter) -> - {selected, [{BCount}]} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT COUNT(*) ", - "FROM mam_muc_message ", Filter]), - mongoose_rdbms:result_to_integer(BCount). - - -%% @doc prepare_filter/5 --spec prepare_filter(RoomID :: mod_mam:archive_id(), Borders :: mod_mam:borders() | undefined, - Start :: unix_timestamp() | undefined, End :: unix_timestamp() | undefined, - WithJID :: jid:jid() | undefined, SearchText :: binary() | undefined) -> filter(). -prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText) -> - SWithNick = maybe_jid_to_escaped_resource(WithJID), - StartID = maybe_encode_compact_uuid(Start, 0), - EndID = maybe_encode_compact_uuid(End, 255), - StartID2 = apply_start_border(Borders, StartID), - EndID2 = apply_end_border(Borders, EndID), - make_filter(RoomID, StartID2, EndID2, SWithNick, SearchText). - - --spec make_filter(RoomID :: non_neg_integer(), - StartID :: mod_mam:message_id() | undefined, - EndID :: mod_mam:message_id() | undefined, - SWithNick :: escaped_jid() | undefined, - SearchText :: binary() | undefined) -> filter(). -make_filter(RoomID, StartID, EndID, SWithNick, SearchText) -> - ["WHERE room_id=", use_escaped_integer(escape_room_id(RoomID)), - case StartID of - undefined -> ""; - _ -> [" AND id >= ", use_escaped_integer(escape_integer(StartID))] - end, - case EndID of - undefined -> ""; - _ -> [" AND id <= ", use_escaped_integer(escape_integer(EndID))] - end, - case SWithNick of - undefined -> ""; - _ -> [" AND nick_name = ", use_escaped_string(SWithNick)] - end, - case SearchText of - undefined -> ""; - _ -> prepare_search_filters(SearchText) - end - ]. - -%% Constructs a separate LIKE filter for each word. -%% SearchText example is "word1%word2%word3". -prepare_search_filters(SearchText) -> - Words = binary:split(SearchText, <<"%">>, [global]), - [prepare_search_filter(Word) || Word <- Words]. - --spec prepare_search_filter(binary()) -> filter(). -prepare_search_filter(Word) -> - [" AND search_body like ", - %% Search for "%Word%" - mongoose_rdbms:use_escaped_like(mongoose_rdbms:escape_like(Word))]. - -%% @doc #rsm_in{ -%% max = non_neg_integer() | undefined, -%% direction = before | aft | undefined, -%% id = binary() | undefined, -%% index = non_neg_integer() | undefined} --spec calc_offset(Host :: jid:server(), - Filter :: filter(), PageSize :: non_neg_integer(), - TotalCount :: non_neg_integer(), RSM :: jlib:rsm_in() | undefined) - -> non_neg_integer(). -calc_offset(_LS, _F, _PS, _TC, #rsm_in{direction = undefined, index = Index}) - when is_integer(Index) -> - Index; -%% Requesting the Last Page in a Result Set -calc_offset(_LS, _F, PS, TC, #rsm_in{direction = before, id = undefined}) -> - max(0, TC - PS); -calc_offset(Host, F, PS, _TC, #rsm_in{direction = before, id = MessID}) - when is_integer(MessID) -> - SMessID = escape_message_id(MessID), - max(0, calc_before(Host, F, SMessID) - PS); -calc_offset(Host, F, _PS, _TC, #rsm_in{direction = aft, id = MessID}) - when is_integer(MessID) -> - SMessID = escape_message_id(MessID), - calc_index(Host, F, SMessID); -calc_offset(_LS, _F, _PS, _TC, _RSM) -> - 0. - - --spec escape_message_id(mod_mam:message_id()) -> escaped_message_id(). -escape_message_id(MessID) when is_integer(MessID) -> - escape_integer(MessID). - - --spec escape_room_id(mod_mam:archive_id()) -> escaped_room_id(). -escape_room_id(RoomID) when is_integer(RoomID) -> - escape_integer(RoomID). - - --spec maybe_jid_to_escaped_resource('undefined' | jid:jid()) - -> 'undefined' | mongoose_rdbms:escaped_string(). -maybe_jid_to_escaped_resource(undefined) -> - undefined; -maybe_jid_to_escaped_resource(#jid{lresource = <<>>}) -> - undefined; -maybe_jid_to_escaped_resource(#jid{lresource = WithLResource}) -> - mongoose_rdbms:escape_string(WithLResource). - - --spec maybe_encode_compact_uuid('undefined' | integer(), 0 | 255) - -> 'undefined' | integer(). -maybe_encode_compact_uuid(undefined, _) -> - undefined; -maybe_encode_compact_uuid(Microseconds, NodeID) -> - encode_compact_uuid(Microseconds, NodeID). +remove_archive(Host, ArcID) -> + {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_muc_archive_remove, [ArcID]). +%% GDPR logic +extract_gdpr_messages(Env, ArcID) -> + Filters = [{equal, room_id, ArcID}], + lookup_query(lookup, Env, Filters, asc). -%% ---------------------------------------------------------------------- -%% Optimizations - -packet_to_stored_binary(Host, Packet) -> - Module = db_message_codec(Host), - mam_message:encode(Module, Packet). - -stored_binary_to_packet(Host, Bin) -> - Module = db_message_codec(Host), - mam_message:decode(Module, Bin). - --spec db_message_codec(Host :: jid:server()) -> module(). -db_message_codec(Host) -> - gen_mod:get_module_opt(Host, ?MODULE, db_message_format, mam_message_compressed_eterm). - - -gdpr_decode_packet(Host, SDataRaw) -> - Codec = mod_mam_meta:get_mam_module_opt(Host, ?MODULE, db_message_format, - mam_message_compressed_eterm), - Data = mongoose_rdbms:unescape_binary(Host, SDataRaw), - Message = mam_message:decode(Codec, Data), - exml:to_binary(Message). +%% Lookup logic +-spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> + {ok, mod_mam:lookup_result()}. +lookup_messages({error, _Reason}=Result, _Host, _Params) -> + Result; +lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> + Env = env_vars(Host, ArcJID), + ExdParams = mam_encoder:extend_lookup_params(Params, Env), + Filter = mam_filter:produce_filter(ExdParams, lookup_fields()), + mam_lookup:lookup(Env, Filter, ExdParams). + +lookup_query(QueryType, Env, Filters, Order) -> + mam_lookup_sql:lookup_query(QueryType, Env, Filters, Order). diff --git a/src/mam/mod_mam_muc_rdbms_async_pool_writer.erl b/src/mam/mod_mam_muc_rdbms_async_pool_writer.erl index 67ac1d3e92..08b6f4efde 100644 --- a/src/mam/mod_mam_muc_rdbms_async_pool_writer.erl +++ b/src/mam/mod_mam_muc_rdbms_async_pool_writer.erl @@ -153,7 +153,8 @@ stop_worker(Proc) -> -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok | {error, timeout}. -archive_message(_Result, Host, Params = #{archive_id := RoomID}) -> +archive_message(_Result, Host, Params0 = #{archive_id := RoomID}) -> + Params = mod_mam_muc_rdbms_arch:extend_params_with_sender_id(Host, Params0), Worker = select_worker(Host, RoomID), WorkerPid = whereis(Worker), %% Send synchronously if queue length is too long. diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index b369474d63..89c30f6a22 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -62,22 +62,22 @@ -spec start(jid:server(), _) -> ok. start(Host, Opts) -> - start_pm(Host, Opts), + start_hooks(Host, Opts), register_prepared_queries(), ok. -spec stop(jid:server()) -> ok. stop(Host) -> - stop_pm(Host). + stop_hooks(Host). -spec get_mam_pm_gdpr_data(ejabberd_gen_mam_archive:mam_pm_gdpr_data(), jid:jid()) -> ejabberd_gen_mam_archive:mam_pm_gdpr_data(). get_mam_pm_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> case mod_mam:archive_id(Host, User) of undefined -> []; - ArchiveID -> + ArcID -> Env = env_vars(Host, ArcJID), - {selected, Rows} = extract_gdpr_messages(Env, ArchiveID), + {selected, Rows} = extract_gdpr_messages(Env, ArcID), [uniform_to_gdpr(row_to_uniform_format(Row, Env)) || Row <- Rows] ++ Acc end. @@ -87,8 +87,8 @@ uniform_to_gdpr({MessID, RemoteJID, Packet}) -> %% ---------------------------------------------------------------------- %% Add hooks for mod_mam --spec start_pm(jid:server(), _) -> 'ok'. -start_pm(Host, _Opts) -> +-spec start_hooks(jid:server(), _) -> ok. +start_hooks(Host, _Opts) -> case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of true -> ok; @@ -102,8 +102,8 @@ start_pm(Host, _Opts) -> ok. --spec stop_pm(jid:server()) -> ok. -stop_pm(Host) -> +-spec stop_hooks(jid:server()) -> ok. +stop_hooks(Host) -> case gen_mod:get_module_opt(Host, ?MODULE, no_writer, false) of true -> ok; @@ -219,8 +219,8 @@ get_retract_id(Packet, #{has_message_retraction := Enabled}) -> -spec archive_size(Size :: integer(), Host :: jid:server(), ArcId :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> integer(). -archive_size(Size, Host, UserID, ArcJID) when is_integer(Size) -> - Filter = [{equal, user_id, UserID}], +archive_size(Size, Host, ArcID, ArcJID) when is_integer(Size) -> + Filter = [{equal, user_id, ArcID}], Env = env_vars(Host, ArcJID), Result = lookup_query(count, Env, Filter, unordered), mongoose_rdbms:selected_to_integer(Result). @@ -230,7 +230,8 @@ archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> try Env = env_vars(Host, ArcJID), do_archive_message(Host, Params, Env), - retract_message(Host, Params, Env) + retract_message(Host, Params, Env), + ok catch Class:Reason:StackTrace -> ?LOG_ERROR(#{what => archive_message_failed, host => Host, mam_params => Params, @@ -250,42 +251,42 @@ retract_message(Host, #{local_jid := ArcJID} = Params) -> retract_message(Host, Params, Env). -spec retract_message(jid:server(), mod_mam:archive_message_params(), env_vars()) -> ok. -retract_message(Host, #{archive_id := UserID, remote_jid := RemJID, +retract_message(Host, #{archive_id := ArcID, remote_jid := RemJID, direction := Dir, packet := Packet}, Env) -> case get_retract_id(Packet, Env) of none -> ok; OriginID -> - Info = get_retraction_info(Host, UserID, RemJID, OriginID, Dir, Env), - make_tombstone(Host, UserID, OriginID, Info, Env) + Info = get_retraction_info(Host, ArcID, RemJID, OriginID, Dir, Env), + make_tombstone(Host, ArcID, OriginID, Info, Env) end. -get_retraction_info(Host, UserID, RemJID, OriginID, Dir, Env) -> +get_retraction_info(Host, ArcID, RemJID, OriginID, Dir, Env) -> %% Code style notice: %% - Add Ext prefix for all externally encoded data %% (in cases, when we usually add Bin, B, S Esc prefixes) ExtBareRemJID = mam_encoder:encode_jid(jid:to_bare(RemJID), Env), ExtDir = mam_encoder:encode_direction(Dir), {selected, Rows} = execute_select_messages_to_retract( - Host, UserID, ExtBareRemJID, OriginID, ExtDir), + Host, ArcID, ExtBareRemJID, OriginID, ExtDir), mam_decoder:decode_retraction_info(Env, Rows). -make_tombstone(_Host, UserID, OriginID, skip, _Env) -> +make_tombstone(_Host, ArcID, OriginID, skip, _Env) -> ?LOG_INFO(#{what => make_tombstone_failed, text => <<"Message to retract was not found by origin id">>, - user_id => UserID, origin_id => OriginID}); -make_tombstone(Host, UserID, OriginID, #{packet := Packet, message_id := MessID}, Env) -> + user_id => ArcID, origin_id => OriginID}); +make_tombstone(Host, ArcID, OriginID, #{packet := Packet, message_id := MessID}, Env) -> Tombstone = mod_mam_utils:tombstone(Packet, OriginID), TombstoneData = mam_encoder:encode_packet(Tombstone, Env), - execute_make_tombstone(Host, TombstoneData, UserID, MessID). + execute_make_tombstone(Host, TombstoneData, ArcID, MessID). -execute_select_messages_to_retract(Host, UserID, BareRemJID, OriginID, Dir) -> +execute_select_messages_to_retract(Host, ArcID, BareRemJID, OriginID, Dir) -> mod_mam_utils:success_sql_execute(Host, mam_select_messages_to_retract, - [UserID, BareRemJID, OriginID, Dir]). + [ArcID, BareRemJID, OriginID, Dir]). -execute_make_tombstone(Host, TombstoneData, UserID, MessID) -> +execute_make_tombstone(Host, TombstoneData, ArcID, MessID) -> {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_make_tombstone, - [TombstoneData, UserID, MessID]). + [TombstoneData, ArcID, MessID]). %% Insert logic -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). @@ -303,18 +304,18 @@ prepare_insert(Name, NumRows) -> %% Removal logic -spec remove_archive(Acc :: mongoose_acc:t(), Host :: jid:server(), - ArchiveID :: mod_mam:archive_id(), + ArcID :: mod_mam:archive_id(), RoomJID :: jid:jid()) -> mongoose_acc:t(). -remove_archive(Acc, Host, UserID, _ArcJID) -> - remove_archive(Host, UserID), +remove_archive(Acc, Host, ArcID, _ArcJID) -> + remove_archive(Host, ArcID), Acc. -remove_archive(Host, UserID) -> - {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_archive_remove, [UserID]). +remove_archive(Host, ArcID) -> + {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_archive_remove, [ArcID]). %% GDPR logic -extract_gdpr_messages(Env, ArchiveID) -> - Filters = [{equal, user_id, ArchiveID}], +extract_gdpr_messages(Env, ArcID) -> + Filters = [{equal, user_id, ArcID}], lookup_query(lookup, Env, Filters, asc). %% Lookup logic From a4fa93440c75b4725a3f96b41736fb4b86cb7b85 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 2 Dec 2020 21:41:26 +0100 Subject: [PATCH 37/92] Address review comments --- src/mam/mam_lookup_sql.erl | 9 ++++----- src/mam/mod_mam_muc_rdbms_arch.erl | 6 +++--- src/mam/mod_mam_rdbms_arch.erl | 6 +++--- src/rdbms/rdbms_queries.erl | 6 ------ 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index 2ce982b0d1..7f532575e4 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -46,13 +46,12 @@ lookup_query(QueryType, Env, Filters, Order) -> reason => Error, host => Host}, ?LOG_ERROR(What), error(What) - catch Class:Error:Stacktrace -> + catch error:Error:Stacktrace -> What = #{what => mam_lookup_failed, statement => StmtName, sql_query => lookup_sql_binary(QueryType, Table, Env, Filters, Order), - class => Class, stacktrace => Stacktrace, - reason => Error, host => Host}, + stacktrace => Stacktrace, reason => Error, host => Host}, ?LOG_ERROR(What), - erlang:raise(Class, Error, Stacktrace) + erlang:raise(error, Error, Stacktrace) end. lookup_sql_binary(QueryType, Table, Env, Filters, Order) -> @@ -110,7 +109,7 @@ skip_undefined(List) -> [X || X <- List, X =/= undefined]. filter_to_sql({Op, Column, _Value}) -> filter_to_sql(atom_to_list(Column), Op). op_to_id(equal) -> "eq"; -op_to_id(lower) -> "lt"; %% lower than +op_to_id(lower) -> "lt"; %% less than op_to_id(greater) -> "gt"; %% greater than op_to_id(le) -> "le"; %% lower or equal op_to_id(ge) -> "ge"; %% greater or equal diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index b28f968f15..951d81976f 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -220,11 +220,11 @@ archive_message(_Result, Host, Params0 = #{local_jid := ArcJID}) -> do_archive_message(Host, Params, Env), retract_message(Host, Params, Env), ok - catch Class:Reason:StackTrace -> + catch error:Reason:StackTrace -> ?LOG_ERROR(#{what => archive_message_failed, host => Host, mam_params => Params0, - class => Class, reason => Reason, stacktrace => StackTrace}), - erlang:raise(Class, Reason, StackTrace) + reason => Reason, stacktrace => StackTrace}), + erlang:raise(error, Reason, StackTrace) end. do_archive_message(Host, Params, Env) -> diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 89c30f6a22..de632fdaea 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -232,11 +232,11 @@ archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> do_archive_message(Host, Params, Env), retract_message(Host, Params, Env), ok - catch Class:Reason:StackTrace -> + catch error:Reason:StackTrace -> ?LOG_ERROR(#{what => archive_message_failed, host => Host, mam_params => Params, - class => Class, reason => Reason, stacktrace => StackTrace}), - erlang:raise(Class, Reason, StackTrace) + reason => Reason, stacktrace => StackTrace}), + erlang:raise(error, Reason, StackTrace) end. do_archive_message(Host, Params, Env) -> diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index c50340dfb3..e46791cadb 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -28,8 +28,6 @@ -export([get_db_type/0, begin_trans/0, - get_db_specific_limits/0, - get_db_specific_limits/1, get_db_specific_limits_binaries/1, get_db_specific_offset/2, add_limit_arg/2, @@ -852,10 +850,6 @@ do_get_db_specific_offset(mssql, Offset, Limit) -> do_get_db_specific_offset(_, Offset, _Limit) -> [" OFFSET ", Offset]. -get_db_specific_limits() -> - LimitStr = "?", - do_get_db_specific_limits(?RDBMS_TYPE, LimitStr, true). - add_limit_arg(Limit, Args) -> add_limit_arg(?RDBMS_TYPE, Limit, Args). From 7f00110e1a5b5361024d29645846a238b5dd8ead Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Dec 2020 10:24:51 +0100 Subject: [PATCH 38/92] Pass Offset and Limit as a separate arg --- src/mam/mam_lookup.erl | 9 ++-- src/mam/mam_lookup_sql.erl | 83 +++++++++++++++++------------- src/mam/mod_mam_muc_rdbms_arch.erl | 10 ++-- src/mam/mod_mam_rdbms_arch.erl | 10 ++-- src/rdbms/rdbms_queries.erl | 16 +++--- 5 files changed, 71 insertions(+), 57 deletions(-) diff --git a/src/mam/mam_lookup.erl b/src/mam/mam_lookup.erl index 82261a6d3a..7ce125f541 100644 --- a/src/mam/mam_lookup.erl +++ b/src/mam/mam_lookup.erl @@ -23,8 +23,8 @@ lookup(Env = #{}, Filter, Params = #{rsm := RSM}) when is_list(Filter) -> OptParams = Params#{opt_count_type => opt_count_type(RSM)}, choose_lookup_messages_strategy(Env, Filter, OptParams). -lookup_query(QueryType, #{lookup_fn := LookupF} = Env, Filters, Order) -> - LookupF(QueryType, Env, Filters, Order). +lookup_query(QueryType, #{lookup_fn := LookupF} = Env, Filters, Order, OffsetLimit) -> + LookupF(QueryType, Env, Filters, Order, OffsetLimit). decode_row(Row, #{decode_row_fn := DecodeF} = Env) -> DecodeF(Row, Env). @@ -174,14 +174,13 @@ maybe_reverse(asc, List) -> List; maybe_reverse(desc, List) -> lists:reverse(List). extract_rows(Env, Filters, Offset, Max, Order) -> - Filters2 = Filters ++ rdbms_queries:limit_offset_filters(Max, Offset), - lookup_query(lookup, Env, Filters2, Order). + lookup_query(lookup, Env, Filters, Order, {Offset, Max}). %% @doc Get the total result set size. %% SELECT COUNT(*) as count FROM mam_message -spec calc_count(env_vars(), filter()) -> non_neg_integer(). calc_count(Env, Filter) -> - Result = lookup_query(count, Env, Filter, unordered), + Result = lookup_query(count, Env, Filter, unordered, all), mongoose_rdbms:selected_to_integer(Result). %% @doc Calculate a zero-based index of the row with UID in the result test. diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index 7f532575e4..641e629cb5 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -1,10 +1,12 @@ %% Makes a SELECT SQL query -module(mam_lookup_sql). --export([lookup_query/4]). +-export([lookup_query/5]). -include("mongoose_logger.hrl"). -include("mongoose_mam.hrl"). +-type offset_limit() :: all | {Offset :: non_neg_integer(), Limit :: non_neg_integer()}. + %% The ONLY usage of Env is in these functions: %% The rest of code should treat Env as opaque (i.e. the code just passes Env around). host(#{host := Host}) -> Host. @@ -22,70 +24,81 @@ column_to_id(#{column_to_id_fn := F}, Col) -> F(Col). %% %% Filters are in format {Op, Column, Value} %% QueryType should be an atom, that we pass into the columns_sql_fn function. --spec lookup_query(QueryType :: atom(), Env :: map(), Filters :: list(), Order :: atom()) -> term(). -lookup_query(QueryType, Env, Filters, Order) -> +-spec lookup_query(QueryType :: atom(), Env :: map(), Filters :: list(), + Order :: atom(), OffsetLimit :: offset_limit()) -> term(). +lookup_query(QueryType, Env, Filters, Order, OffsetLimit) -> Table = table(Env), Host = host(Env), - StmtName = filters_to_statement_name(Env, QueryType, Table, Filters, Order), + StmtName = filters_to_statement_name(Env, QueryType, Table, Filters, Order, OffsetLimit), case mongoose_rdbms:prepared(StmtName) of false -> %% Create a new type of a query - SQL = lookup_sql_binary(QueryType, Table, Env, Filters, Order), - Columns = filters_to_columns(Filters), + SQL = lookup_sql_binary(QueryType, Table, Env, Filters, Order, OffsetLimit), + Columns = filters_to_columns(Filters, OffsetLimit), mongoose_rdbms:prepare(StmtName, Table, Columns, SQL); true -> ok end, - Args = filters_to_args(Filters), + Args = filters_to_args(Filters, OffsetLimit), try mongoose_rdbms:execute(Host, StmtName, Args) of {selected, Rs} when is_list(Rs) -> {selected, Rs}; Error -> What = #{what => mam_lookup_failed, statement => StmtName, - sql_query => lookup_sql_binary(QueryType, Table, Env, Filters, Order), + sql_query => lookup_sql_binary(QueryType, Table, Env, Filters, Order, OffsetLimit), reason => Error, host => Host}, ?LOG_ERROR(What), error(What) catch error:Error:Stacktrace -> What = #{what => mam_lookup_failed, statement => StmtName, - sql_query => lookup_sql_binary(QueryType, Table, Env, Filters, Order), + sql_query => lookup_sql_binary(QueryType, Table, Env, Filters, Order, OffsetLimit), stacktrace => Stacktrace, reason => Error, host => Host}, ?LOG_ERROR(What), erlang:raise(error, Error, Stacktrace) end. -lookup_sql_binary(QueryType, Table, Env, Filters, Order) -> - iolist_to_binary(lookup_sql(QueryType, Table, Env, Filters, Order)). +lookup_sql_binary(QueryType, Table, Env, Filters, Order, OffsetLimit) -> + iolist_to_binary(lookup_sql(QueryType, Table, Env, Filters, Order, OffsetLimit)). -lookup_sql(QueryType, Table, Env, Filters, Order) -> +lookup_sql(QueryType, Table, Env, Filters, Order, OffsetLimit) -> IndexHintSQL = index_hint_sql(Env), FilterSQL = filters_to_sql(Filters), OrderSQL = order_to_sql(Order), - LimitSQL = limit_sql(Filters), - ["SELECT ", columns_sql(Env, QueryType), " FROM ", atom_to_list(Table), " ", + {LimitSQL, TopSQL} = limit_sql(OffsetLimit), + ["SELECT ", TopSQL, " ", columns_sql(Env, QueryType), + " FROM ", atom_to_list(Table), " ", IndexHintSQL, FilterSQL, OrderSQL, LimitSQL]. -%% Caller should provide both limit and offset fields in the correct order. -%% See limit_offset_filters. -%% No limits option is fine too (it is used with count and GDPR). -limit_sql(Filters) -> - HasLimit = lists:keymember(limit, 1, Filters), %% and offset - case HasLimit of - true -> rdbms_queries:limit_offset_sql(); - false -> "" - end. +limit_sql(all) -> {"", ""}; +limit_sql({0, Limit}) -> rdbms_queries:get_db_specific_limits(); +limit_sql({_Offset, _Limit}) -> {rdbms_queries:limit_offset_sql(), ""}. + +filters_to_columns(Filters, OffsetLimit) -> + limit_offset_to_columns(OffsetLimit, [Column || {_Op, Column, _Value} <- Filters]). -filters_to_columns(Filters) -> - [Column || {_Op, Column, _Value} <- Filters]. +filters_to_args(Filters, OffsetLimit) -> + limit_offset_to_args(OffsetLimit, [Value || {_Op, _Column, Value} <- Filters]). -filters_to_args(Filters) -> - [Value || {_Op, _Column, Value} <- Filters]. +limit_offset_to_args(all, Args) -> + Args; +limit_offset_to_args({0, Limit}, Args) -> + rdbms_queries:add_limit_arg(Limit, Args); +limit_offset_to_args({Offset, Limit}, Args) -> + Args ++ rdbms_queries:limit_offset_args(Limit, Offset). -filters_to_statement_name(Env, QueryType, Table, Filters, Order) -> +limit_offset_to_columns(all, Columns) -> + Columns; +limit_offset_to_columns({0, _Limit}, Columns) -> + rdbms_queries:add_limit_arg(limit, Columns); +limit_offset_to_columns({_Offset, _Limit}, Columns) -> + Columns ++ rdbms_queries:limit_offset_args(limit, offset). + +filters_to_statement_name(Env, QueryType, Table, Filters, Order, OffsetLimit) -> QueryId = query_type_to_id(QueryType), Ids = [op_to_id(Op) ++ column_to_id(Env, Col) || {Op, Col, _Val} <- Filters], OrderId = order_type_to_id(Order), - list_to_atom(atom_to_list(Table) ++ "_" ++ QueryId ++ "_" ++ OrderId ++ "_" ++ lists:append(Ids)). + LimitId = offset_limit_to_id(OffsetLimit), + list_to_atom(atom_to_list(Table) ++ "_" ++ QueryId ++ "_" ++ OrderId ++ "_" ++ lists:append(Ids) ++ "_" ++ LimitId). query_type_to_id(QueryType) -> atom_to_list(QueryType). @@ -97,6 +110,10 @@ order_to_sql(asc) -> " ORDER BY id "; order_to_sql(desc) -> " ORDER BY id DESC "; order_to_sql(unordered) -> " ". +offset_limit_to_id({0, _Limit}) -> "limit"; +offset_limit_to_id({_Offset, _Limit}) -> "offlim"; +offset_limit_to_id(all) -> "all". + filters_to_sql(Filters) -> SQLs = [filter_to_sql(Filter) || Filter <- Filters], case skip_undefined(SQLs) of @@ -113,15 +130,11 @@ op_to_id(lower) -> "lt"; %% less than op_to_id(greater) -> "gt"; %% greater than op_to_id(le) -> "le"; %% lower or equal op_to_id(ge) -> "ge"; %% greater or equal -op_to_id(like) -> "lk"; -op_to_id(limit) -> "li"; -op_to_id(offset) -> "of". +op_to_id(like) -> "lk". filter_to_sql(Column, equal) -> Column ++ " = ?"; filter_to_sql(Column, lower) -> Column ++ " < ?"; filter_to_sql(Column, greater) -> Column ++ " > ?"; filter_to_sql(Column, le) -> Column ++ " <= ?"; filter_to_sql(Column, ge) -> Column ++ " >= ?"; -filter_to_sql(Column, like) -> Column ++ " LIKE ?"; -filter_to_sql(_Column, limit) -> undefined; -filter_to_sql(_Column, offset) -> undefined. +filter_to_sql(Column, like) -> Column ++ " LIKE ?". diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 951d81976f..172cde33b8 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -154,7 +154,7 @@ env_vars(Host, ArcJID) -> index_hint_fn => fun index_hint_sql/1, columns_sql_fn => fun columns_sql/1, column_to_id_fn => fun column_to_id/1, - lookup_fn => fun lookup_query/4, + lookup_fn => fun lookup_query/5, decode_row_fn => fun row_to_uniform_format/2, has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam_muc, Host), has_full_text_search => mod_mam_utils:has_full_text_search(mod_mam_muc, Host), @@ -204,7 +204,7 @@ get_retract_id(Packet, #{has_message_retraction := Enabled}) -> archive_size(Size, Host, ArcID, ArcJID) when is_integer(Size) -> Filter = [{equal, room_id, ArcID}], Env = env_vars(Host, ArcJID), - Result = lookup_query(count, Env, Filter, unordered), + Result = lookup_query(count, Env, Filter, unordered, all), mongoose_rdbms:selected_to_integer(Result). extend_params_with_sender_id(Host, Params = #{remote_jid := SenderJID}) -> @@ -299,7 +299,7 @@ remove_archive(Host, ArcID) -> %% GDPR logic extract_gdpr_messages(Env, ArcID) -> Filters = [{equal, room_id, ArcID}], - lookup_query(lookup, Env, Filters, asc). + lookup_query(lookup, Env, Filters, asc, all). %% Lookup logic -spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> @@ -312,5 +312,5 @@ lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> Filter = mam_filter:produce_filter(ExdParams, lookup_fields()), mam_lookup:lookup(Env, Filter, ExdParams). -lookup_query(QueryType, Env, Filters, Order) -> - mam_lookup_sql:lookup_query(QueryType, Env, Filters, Order). +lookup_query(QueryType, Env, Filters, Order, OffsetLimit) -> + mam_lookup_sql:lookup_query(QueryType, Env, Filters, Order, OffsetLimit). diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index de632fdaea..82f1abc474 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -167,7 +167,7 @@ env_vars(Host, ArcJID) -> index_hint_fn => fun index_hint_sql/1, columns_sql_fn => fun columns_sql/1, column_to_id_fn => fun column_to_id/1, - lookup_fn => fun lookup_query/4, + lookup_fn => fun lookup_query/5, decode_row_fn => fun row_to_uniform_format/2, has_message_retraction => mod_mam_utils:has_message_retraction(mod_mam, Host), has_full_text_search => mod_mam_utils:has_full_text_search(mod_mam, Host), @@ -222,7 +222,7 @@ get_retract_id(Packet, #{has_message_retraction := Enabled}) -> archive_size(Size, Host, ArcID, ArcJID) when is_integer(Size) -> Filter = [{equal, user_id, ArcID}], Env = env_vars(Host, ArcJID), - Result = lookup_query(count, Env, Filter, unordered), + Result = lookup_query(count, Env, Filter, unordered, all), mongoose_rdbms:selected_to_integer(Result). -spec archive_message(_Result, jid:server(), mod_mam:archive_message_params()) -> ok. @@ -316,7 +316,7 @@ remove_archive(Host, ArcID) -> %% GDPR logic extract_gdpr_messages(Env, ArcID) -> Filters = [{equal, user_id, ArcID}], - lookup_query(lookup, Env, Filters, asc). + lookup_query(lookup, Env, Filters, asc, all). %% Lookup logic -spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> @@ -329,5 +329,5 @@ lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> Filter = mam_filter:produce_filter(ExdParams, lookup_fields()), mam_lookup:lookup(Env, Filter, ExdParams). -lookup_query(QueryType, Env, Filters, Order) -> - mam_lookup_sql:lookup_query(QueryType, Env, Filters, Order). +lookup_query(QueryType, Env, Filters, Order, OffsetLimit) -> + mam_lookup_sql:lookup_query(QueryType, Env, Filters, Order, OffsetLimit). diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index e46791cadb..2db3da8f9d 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -28,11 +28,12 @@ -export([get_db_type/0, begin_trans/0, + get_db_specific_limits/0, get_db_specific_limits_binaries/1, get_db_specific_offset/2, add_limit_arg/2, limit_offset_sql/0, - limit_offset_filters/2, + limit_offset_args/2, sql_transaction/2, get_last/2, select_last/3, @@ -822,6 +823,9 @@ create_bulk_insert_query(Table, Fields, RowsNum) when RowsNum > 0 -> Fields2 = lists:append(lists:duplicate(RowsNum, Fields)), {Sql, Fields2}. +get_db_specific_limits() -> + do_get_db_specific_limits(?RDBMS_TYPE, "?", true). + -spec get_db_specific_limits(integer()) -> {SQL :: nonempty_string(), []} | {[], MSSQL::nonempty_string()}. get_db_specific_limits(Limit) -> @@ -870,10 +874,8 @@ limit_offset_sql(mssql) -> limit_offset_sql(_) -> <<" LIMIT ? OFFSET ?">>. -limit_offset_filters(Limit, Offset) -> - limit_offset_filters(?RDBMS_TYPE, Limit, Offset). +limit_offset_args(Limit, Offset) -> + limit_offset_args(?RDBMS_TYPE, Limit, Offset). -limit_offset_filters(mssql, Limit, Offset) -> - [{offset, offset, Offset}, {limit, limit, Limit}]; -limit_offset_filters(_, Limit, Offset) -> - [{limit, limit, Limit}, {offset, offset, Offset}]. +limit_offset_args(mssql, Limit, Offset) -> [Offset, Limit]; +limit_offset_args(_, Limit, Offset) -> [Limit, Offset]. From c8de1640e08f489b6539cc8e8fcd301edff2e000 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Dec 2020 11:26:01 +0100 Subject: [PATCH 39/92] Fix GDPR MAM MUC query --- src/mam/mam_decoder.erl | 5 +++++ src/mam/mam_lookup_sql.erl | 2 +- src/mam/mod_mam_muc_rdbms_arch.erl | 27 ++++++++++++++------------- src/mam/mod_mam_rdbms_arch.erl | 3 ++- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/mam/mam_decoder.erl b/src/mam/mam_decoder.erl index 3647b002ae..e38c0e664a 100644 --- a/src/mam/mam_decoder.erl +++ b/src/mam/mam_decoder.erl @@ -1,6 +1,7 @@ -module(mam_decoder). -export([decode_row/2]). -export([decode_muc_row/2]). +-export([decode_muc_gdpr_row/2]). -export([decode_retraction_info/2]). -type env_vars() :: map(). @@ -17,6 +18,10 @@ decode_muc_row({ExtMessID, Nick, ExtData}, Env = #{archive_jid := RoomJID}) -> Packet = decode_packet(ExtData, Env), {MessID, SrcJID, Packet}. +decode_muc_gdpr_row({ExtMessID, ExtData}, Env) -> + Packet = decode_packet(ExtData, Env), + {ExtMessID, Packet}. + decode_retraction_info(_Env, []) -> skip; decode_retraction_info(Env, [{ResMessID, Data}]) -> Packet = decode_packet(Data, Env), diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index 641e629cb5..6aad931f51 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -70,7 +70,7 @@ lookup_sql(QueryType, Table, Env, Filters, Order, OffsetLimit) -> IndexHintSQL, FilterSQL, OrderSQL, LimitSQL]. limit_sql(all) -> {"", ""}; -limit_sql({0, Limit}) -> rdbms_queries:get_db_specific_limits(); +limit_sql({0, _Limit}) -> rdbms_queries:get_db_specific_limits(); limit_sql({_Offset, _Limit}) -> {rdbms_queries:limit_offset_sql(), ""}. filters_to_columns(Filters, OffsetLimit) -> diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 172cde33b8..43aae19d9c 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -62,18 +62,17 @@ stop(Host) -> -spec get_mam_muc_gdpr_data(ejabberd_gen_mam_archive:mam_pm_gdpr_data(), jid:jid()) -> ejabberd_gen_mam_archive:mam_pm_gdpr_data(). -get_mam_muc_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> +get_mam_muc_gdpr_data(Acc, #jid{luser = User, lserver = Host} = _UserJID) -> case mod_mam:archive_id(Host, User) of - undefined -> []; - ArcID -> - Env = env_vars(Host, ArcJID), - {selected, Rows} = extract_gdpr_messages(Env, ArcID), - [uniform_to_gdpr(row_to_uniform_format(Row, Env)) || Row <- Rows] ++ Acc + undefined -> + Acc; + SenderID -> + %% We don't know the real room JID here, use FakeEnv + FakeEnv = env_vars(Host, jid:make(<<>>, <<>>, <<>>)), + {selected, Rows} = extract_gdpr_messages(Host, SenderID), + [mam_decoder:decode_muc_gdpr_row(Row, FakeEnv) || Row <- Rows] ++ Acc end. -uniform_to_gdpr({MessID, _Nick, Packet}) -> - {integer_to_binary(MessID), exml:to_binary(Packet)}. - %% ---------------------------------------------------------------------- %% Add hooks for mod_mam @@ -124,7 +123,10 @@ register_prepared_queries() -> " id, message FROM mam_muc_message" " WHERE room_id = ? AND sender_id = ? " " AND origin_id = ?" - " ORDER BY id DESC ", LimitSQL/binary>>]). + " ORDER BY id DESC ", LimitSQL/binary>>]), + mongoose_rdbms:prepare(mam_muc_extract_gdpr_messages, mam_muc_message, [id, message], + [<<"SELECT id, message FROM mam_muc_message " + " WHERE sender_id = ?">>]). %% ---------------------------------------------------------------------- %% Declarative logic @@ -297,9 +299,8 @@ remove_archive(Host, ArcID) -> {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_muc_archive_remove, [ArcID]). %% GDPR logic -extract_gdpr_messages(Env, ArcID) -> - Filters = [{equal, room_id, ArcID}], - lookup_query(lookup, Env, Filters, asc, all). +extract_gdpr_messages(Host, SenderID) -> + mod_mam_utils:success_sql_execute(Host, mam_muc_extract_gdpr_messages, [SenderID]). %% Lookup logic -spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 82f1abc474..4a6a619548 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -74,7 +74,8 @@ stop(Host) -> ejabberd_gen_mam_archive:mam_pm_gdpr_data(). get_mam_pm_gdpr_data(Acc, #jid{luser = User, lserver = Host} = ArcJID) -> case mod_mam:archive_id(Host, User) of - undefined -> []; + undefined -> + Acc; ArcID -> Env = env_vars(Host, ArcJID), {selected, Rows} = extract_gdpr_messages(Env, ArcID), From 1a2ad373f15b23f49d14a2aa7e76b733d1cf7c66 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Dec 2020 11:55:04 +0100 Subject: [PATCH 40/92] Improve type specs --- src/mam/mam_filter.erl | 8 +++++--- src/mam/mam_lookup.erl | 2 +- src/mam/mam_lookup_sql.erl | 15 +++++++++++++++ src/mam/mod_mam.erl | 4 +++- src/mam/mod_mam_muc_rdbms_arch.erl | 2 +- src/mam/mod_mam_rdbms_arch.erl | 14 ++------------ 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/mam/mam_filter.erl b/src/mam/mam_filter.erl index 929daf3fa3..6e9b8cd12b 100644 --- a/src/mam/mam_filter.erl +++ b/src/mam/mam_filter.erl @@ -10,16 +10,18 @@ | {ge, column(), integer()} | {equal, column(), integer() | binary()} | {lower, column(), integer()} - | {greater, column(), integer()} - | {limit, limit, integer()} - | {offset, offset, integer()}. + | {greater, column(), integer()}. + -type filter() :: [filter_field()]. +-type fields() :: [#lookup_field{}]. +-type params() :: map(). -export_type([filter_field/0]). -export_type([filter/0]). -define(SEARCH_WORDS_LIMIT, 10). +-spec produce_filter(params(), fields()) -> list(filter_field()). produce_filter(Params, Fields) -> [new_filter(Field, Value) || Field <- Fields, diff --git a/src/mam/mam_lookup.erl b/src/mam/mam_lookup.erl index 7ce125f541..1cd510f4a8 100644 --- a/src/mam/mam_lookup.erl +++ b/src/mam/mam_lookup.erl @@ -6,7 +6,7 @@ -include("jlib.hrl"). -include("mongoose_rsm.hrl"). --type filter() :: list(tuple()). +-type filter() :: mam_filter:filter(). -type env_vars() :: map(). -type params() :: map(). -type message_id() :: mod_mam:message_id(). diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index 6aad931f51..0d964af031 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -6,15 +6,29 @@ -include("mongoose_mam.hrl"). -type offset_limit() :: all | {Offset :: non_neg_integer(), Limit :: non_neg_integer()}. +-type sql_part() :: iolist() | binary(). +-type env_vars() :: map(). +-type query_type() :: atom(). +-type column() :: atom(). %% The ONLY usage of Env is in these functions: %% The rest of code should treat Env as opaque (i.e. the code just passes Env around). +-spec host(env_vars()) -> jid:lserver(). host(#{host := Host}) -> Host. + +-spec table(env_vars()) -> atom(). table(#{table := Table}) -> Table. + +-spec index_hint_sql(env_vars()) -> sql_part(). index_hint_sql(Env = #{index_hint_fn := F}) -> F(Env). + +-spec columns_sql(env_vars(), query_type()) -> sql_part(). columns_sql(#{columns_sql_fn := F}, QueryType) -> F(QueryType). + +-spec column_to_id(env_vars(), column()) -> string(). column_to_id(#{column_to_id_fn := F}, Col) -> F(Col). + %% This function uses some fields from Env: %% - host %% - table @@ -123,6 +137,7 @@ filters_to_sql(Filters) -> skip_undefined(List) -> [X || X <- List, X =/= undefined]. +-spec filter_to_sql(mam_filter:filter_field()) -> sql_part(). filter_to_sql({Op, Column, _Value}) -> filter_to_sql(atom_to_list(Column), Op). op_to_id(equal) -> "eq"; diff --git a/src/mam/mod_mam.erl b/src/mam/mod_mam.erl index d10a633163..1d7392630a 100644 --- a/src/mam/mod_mam.erl +++ b/src/mam/mod_mam.erl @@ -142,7 +142,9 @@ source_jid := jid:jid(), origin_id := binary() | none, direction := atom(), - packet := exml:element()}. + packet := exml:element(), + %% Only in mod_mam_muc_rdbms_arch:retract_message/2 + sender_id => mod_mam:archive_id()}. -export_type([rewriter_fun/0, borders/0, diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 43aae19d9c..48a6ed2c2b 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -61,7 +61,7 @@ stop(Host) -> stop_hooks(Host). -spec get_mam_muc_gdpr_data(ejabberd_gen_mam_archive:mam_pm_gdpr_data(), jid:jid()) -> - ejabberd_gen_mam_archive:mam_pm_gdpr_data(). + ejabberd_gen_mam_archive:mam_muc_gdpr_data(). get_mam_muc_gdpr_data(Acc, #jid{luser = User, lserver = Host} = _UserJID) -> case mod_mam:archive_id(Host, User) of undefined -> diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 4a6a619548..90ab56af93 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -42,18 +42,6 @@ %% ---------------------------------------------------------------------- %% Types --type column() :: atom(). --type filter_field() :: {like, column(), binary()} - | {le, column(), integer()} - | {ge, column(), integer()} - | {equal, column(), integer() | binary()} - | {lower, column(), integer()} - | {greater, column(), integer()} - | {limit, limit, integer()} - | {offset, offset, integer()}. - --type filter() :: [filter_field()]. - -type env_vars() :: map(). %% ---------------------------------------------------------------------- @@ -141,6 +129,7 @@ register_prepared_queries() -> %% Declarative logic db_mappings() -> + %% One entry per the database field [#db_mapping{column = id, param = message_id, format = int}, #db_mapping{column = user_id, param = archive_id, format = int}, #db_mapping{column = remote_bare_jid, param = remote_jid, format = bare_jid}, @@ -152,6 +141,7 @@ db_mappings() -> #db_mapping{column = search_body, param = packet, format = search}]. lookup_fields() -> + %% Describe each possible filtering option [#lookup_field{op = equal, column = user_id, param = archive_id, required = true}, #lookup_field{op = ge, column = id, param = start_id}, #lookup_field{op = le, column = id, param = end_id}, From d66e99501fe211e4e033981a9e22041b280e3c70 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Dec 2020 12:09:31 +0100 Subject: [PATCH 41/92] Remove unused column ids --- include/mongoose_mam.hrl | 4 ++-- src/mam/mam_lookup_sql.erl | 2 +- src/mam/mod_mam_muc_rdbms_arch.erl | 6 +----- src/mam/mod_mam_rdbms_arch.erl | 6 ++---- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/include/mongoose_mam.hrl b/include/mongoose_mam.hrl index 4a34faba7c..1279086313 100644 --- a/include/mongoose_mam.hrl +++ b/include/mongoose_mam.hrl @@ -1,2 +1,2 @@ --record(db_mapping, {column, param, format}). --record(lookup_field, {op, column, param, required, value_maker}). +-record(db_mapping, {column :: atom(), param :: atom(), format :: atom()}). +-record(lookup_field, {op :: atom(), column :: atom(), param :: atom(), required :: atom(), value_maker :: atom()}). diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index 0d964af031..c6d06648de 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -143,7 +143,7 @@ filter_to_sql({Op, Column, _Value}) -> filter_to_sql(atom_to_list(Column), Op). op_to_id(equal) -> "eq"; op_to_id(lower) -> "lt"; %% less than op_to_id(greater) -> "gt"; %% greater than -op_to_id(le) -> "le"; %% lower or equal +op_to_id(le) -> "le"; %% less or equal op_to_id(ge) -> "ge"; %% greater or equal op_to_id(like) -> "lk". diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 48a6ed2c2b..9135ecb1fd 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -43,7 +43,6 @@ %% ---------------------------------------------------------------------- %% Types --type column() :: atom(). -type env_vars() :: map(). %% ---------------------------------------------------------------------- @@ -175,10 +174,7 @@ columns_sql(count) -> "COUNT(*)". column_to_id(id) -> "i"; column_to_id(room_id) -> "u"; column_to_id(nick_name) -> "n"; -column_to_id(search_body) -> "s"; -%% fictional columns -column_to_id(limit) -> "l"; -column_to_id(offset) -> "o". +column_to_id(search_body) -> "s". column_names(Mappings) -> [Column || #db_mapping{column = Column} <- Mappings]. diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 90ab56af93..9302b1d442 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -178,14 +178,12 @@ index_hint_sql(#{host := Host}) -> columns_sql(lookup) -> "id, from_jid, message"; columns_sql(count) -> "COUNT(*)". +%% For each unique column in lookup_fields() column_to_id(id) -> "i"; column_to_id(user_id) -> "u"; column_to_id(remote_bare_jid) -> "b"; column_to_id(remote_resource) -> "r"; -column_to_id(search_body) -> "s"; -%% fictional columns -column_to_id(limit) -> "l"; -column_to_id(offset) -> "o". +column_to_id(search_body) -> "s". column_names(Mappings) -> [Column || #db_mapping{column = Column} <- Mappings]. From 03d94ceec1202583d7e37304b45a25ca0e4f54e5 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Dec 2020 12:12:41 +0100 Subject: [PATCH 42/92] Order MAM MUC GDPR messages by ID --- src/mam/mod_mam_muc_rdbms_arch.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 9135ecb1fd..e9e9b0cfcf 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -125,7 +125,7 @@ register_prepared_queries() -> " ORDER BY id DESC ", LimitSQL/binary>>]), mongoose_rdbms:prepare(mam_muc_extract_gdpr_messages, mam_muc_message, [id, message], [<<"SELECT id, message FROM mam_muc_message " - " WHERE sender_id = ?">>]). + " WHERE sender_id = ? ORDER BY id">>]). %% ---------------------------------------------------------------------- %% Declarative logic From 393f46b3da6af041ff4b0581bd174e5a3c43bb70 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Dec 2020 12:16:35 +0100 Subject: [PATCH 43/92] Rename limit_offset_to_args to offset_limit_to_args --- src/mam/mam_lookup_sql.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index c6d06648de..2a0f6e9a79 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -88,23 +88,23 @@ limit_sql({0, _Limit}) -> rdbms_queries:get_db_specific_limits(); limit_sql({_Offset, _Limit}) -> {rdbms_queries:limit_offset_sql(), ""}. filters_to_columns(Filters, OffsetLimit) -> - limit_offset_to_columns(OffsetLimit, [Column || {_Op, Column, _Value} <- Filters]). + offset_limit_to_columns(OffsetLimit, [Column || {_Op, Column, _Value} <- Filters]). filters_to_args(Filters, OffsetLimit) -> - limit_offset_to_args(OffsetLimit, [Value || {_Op, _Column, Value} <- Filters]). + offset_limit_to_args(OffsetLimit, [Value || {_Op, _Column, Value} <- Filters]). -limit_offset_to_args(all, Args) -> +offset_limit_to_args(all, Args) -> Args; -limit_offset_to_args({0, Limit}, Args) -> +offset_limit_to_args({0, Limit}, Args) -> rdbms_queries:add_limit_arg(Limit, Args); -limit_offset_to_args({Offset, Limit}, Args) -> +offset_limit_to_args({Offset, Limit}, Args) -> Args ++ rdbms_queries:limit_offset_args(Limit, Offset). -limit_offset_to_columns(all, Columns) -> +offset_limit_to_columns(all, Columns) -> Columns; -limit_offset_to_columns({0, _Limit}, Columns) -> +offset_limit_to_columns({0, _Limit}, Columns) -> rdbms_queries:add_limit_arg(limit, Columns); -limit_offset_to_columns({_Offset, _Limit}, Columns) -> +offset_limit_to_columns({_Offset, _Limit}, Columns) -> Columns ++ rdbms_queries:limit_offset_args(limit, offset). filters_to_statement_name(Env, QueryType, Table, Filters, Order, OffsetLimit) -> From 553eadbe5e0217e3bee2fee54bf715f4da4bfd3c Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Dec 2020 12:18:42 +0100 Subject: [PATCH 44/92] Fix mam_muc_extract_gdpr_messages on MSSQL --- src/mam/mod_mam_muc_rdbms_arch.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index e9e9b0cfcf..358a01ef1f 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -123,7 +123,7 @@ register_prepared_queries() -> " WHERE room_id = ? AND sender_id = ? " " AND origin_id = ?" " ORDER BY id DESC ", LimitSQL/binary>>]), - mongoose_rdbms:prepare(mam_muc_extract_gdpr_messages, mam_muc_message, [id, message], + mongoose_rdbms:prepare(mam_muc_extract_gdpr_messages, mam_muc_message, [sender_id], [<<"SELECT id, message FROM mam_muc_message " " WHERE sender_id = ? ORDER BY id">>]). From 8276e3cace65e7e793e58cbd3a2754727c587983 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 3 Dec 2020 12:44:09 +0100 Subject: [PATCH 45/92] Define mod_mam_rdbms_arch:env_vars() type --- src/mam/mam_decoder.erl | 2 +- src/mam/mam_encoder.erl | 2 +- src/mam/mam_lookup.erl | 2 +- src/mam/mam_lookup_sql.erl | 9 ++++++++- src/mam/mod_mam_muc_rdbms_arch.erl | 2 +- src/mam/mod_mam_rdbms_arch.erl | 18 +++++++++++++++++- 6 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/mam/mam_decoder.erl b/src/mam/mam_decoder.erl index e38c0e664a..87055693f7 100644 --- a/src/mam/mam_decoder.erl +++ b/src/mam/mam_decoder.erl @@ -4,7 +4,7 @@ -export([decode_muc_gdpr_row/2]). -export([decode_retraction_info/2]). --type env_vars() :: map(). +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). decode_row({ExtMessID, ExtSrcJID, ExtData}, Env) -> MessID = mongoose_rdbms:result_to_integer(ExtMessID), diff --git a/src/mam/mam_encoder.erl b/src/mam/mam_encoder.erl index 71c534f08b..aa0a3196cc 100644 --- a/src/mam/mam_encoder.erl +++ b/src/mam/mam_encoder.erl @@ -6,7 +6,7 @@ -export([extend_lookup_params/2]). -type value_type() :: int | maybe_string | direction | bare_jid | jid | jid_resource | xml | search. --type env_vars() :: map(). +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). -include("mongoose.hrl"). -include("jlib.hrl"). diff --git a/src/mam/mam_lookup.erl b/src/mam/mam_lookup.erl index 1cd510f4a8..03ef2ec40f 100644 --- a/src/mam/mam_lookup.erl +++ b/src/mam/mam_lookup.erl @@ -7,7 +7,7 @@ -include("mongoose_rsm.hrl"). -type filter() :: mam_filter:filter(). --type env_vars() :: map(). +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). -type params() :: map(). -type message_id() :: mod_mam:message_id(). -type maybe_rsm() :: #rsm_in{} | undefined. diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index 2a0f6e9a79..b8df0065bd 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -7,9 +7,16 @@ -type offset_limit() :: all | {Offset :: non_neg_integer(), Limit :: non_neg_integer()}. -type sql_part() :: iolist() | binary(). --type env_vars() :: map(). +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). -type query_type() :: atom(). -type column() :: atom(). +-type lookup_query_fn() :: fun((QueryType :: atom(), Env :: map(), Filters :: list(), + Order :: atom(), OffsetLimit :: offset_limit()) -> term()). + +-export_type([sql_part/0]). +-export_type([query_type/0]). +-export_type([column/0]). +-export_type([lookup_query_fn/0]). %% The ONLY usage of Env is in these functions: %% The rest of code should treat Env as opaque (i.e. the code just passes Env around). diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 358a01ef1f..3abb1f640d 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -43,7 +43,7 @@ %% ---------------------------------------------------------------------- %% Types --type env_vars() :: map(). +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). %% ---------------------------------------------------------------------- %% gen_mod callbacks diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 9302b1d442..cc77fb1bc0 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -42,7 +42,22 @@ %% ---------------------------------------------------------------------- %% Types --type env_vars() :: map(). +-type env_vars() :: #{ + host := jid:lserver(), + archive_jid := jid:jid(), + table := atom(), + index_hint_fn := fun((env_vars()) -> mam_lookup_sql:sql_part()), + columns_sql_fn := fun((mam_lookup_sql:query_type()) -> mam_lookup_sql:sql_part()), + column_to_id_fn := fun((mam_lookup_sql:column()) -> string()), + lookup_fn := mam_lookup_sql:lookup_query_fn(), + decode_row_fn := fun((Row :: tuple(), env_vars()) -> Decoded :: tuple()), + has_message_retraction := boolean(), + has_full_text_search := boolean(), + db_jid_codec := module(), + db_message_codec := module() + }. + +-export_type([env_vars/0]). %% ---------------------------------------------------------------------- %% gen_mod callbacks @@ -149,6 +164,7 @@ lookup_fields() -> #lookup_field{op = equal, column = remote_resource, param = remote_resource}, #lookup_field{op = like, column = search_body, param = norm_search_text, value_maker = search_words}]. +-spec env_vars(jid:lserver(), jid:jid()) -> env_vars(). env_vars(Host, ArcJID) -> %% Please, minimize the usage of the host field. %% It's only for passing into RDBMS. From 6e621c8e91c570008b8c334011b20e108c99465c Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 9 Dec 2020 21:02:30 +0100 Subject: [PATCH 46/92] Replace lower with less --- src/mam/mam_filter.erl | 2 +- src/mam/mam_lookup.erl | 2 +- src/mam/mam_lookup_sql.erl | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mam/mam_filter.erl b/src/mam/mam_filter.erl index 6e9b8cd12b..4bb1edc3d6 100644 --- a/src/mam/mam_filter.erl +++ b/src/mam/mam_filter.erl @@ -9,7 +9,7 @@ | {le, column(), integer()} | {ge, column(), integer()} | {equal, column(), integer() | binary()} - | {lower, column(), integer()} + | {less, column(), integer()} | {greater, column(), integer()}. -type filter() :: [filter_field()]. diff --git a/src/mam/mam_lookup.erl b/src/mam/mam_lookup.erl index 03ef2ec40f..6137f26bec 100644 --- a/src/mam/mam_lookup.erl +++ b/src/mam/mam_lookup.erl @@ -226,7 +226,7 @@ after_id(ID, Filter) -> -spec before_id(message_id(), filter()) -> filter(). before_id(ID, Filter) -> - [{lower, id, ID}|Filter]. + [{less, id, ID}|Filter]. -spec from_id(message_id(), filter()) -> filter(). from_id(ID, Filter) -> diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index b8df0065bd..e7e39bf631 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -148,14 +148,14 @@ skip_undefined(List) -> [X || X <- List, X =/= undefined]. filter_to_sql({Op, Column, _Value}) -> filter_to_sql(atom_to_list(Column), Op). op_to_id(equal) -> "eq"; -op_to_id(lower) -> "lt"; %% less than +op_to_id(less) -> "lt"; %% less than op_to_id(greater) -> "gt"; %% greater than op_to_id(le) -> "le"; %% less or equal op_to_id(ge) -> "ge"; %% greater or equal op_to_id(like) -> "lk". filter_to_sql(Column, equal) -> Column ++ " = ?"; -filter_to_sql(Column, lower) -> Column ++ " < ?"; +filter_to_sql(Column, less) -> Column ++ " < ?"; filter_to_sql(Column, greater) -> Column ++ " > ?"; filter_to_sql(Column, le) -> Column ++ " <= ?"; filter_to_sql(Column, ge) -> Column ++ " >= ?"; From 2ad8ccdb617cb2867a105f2e61b8350e8acaf7c6 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 9 Dec 2020 21:04:04 +0100 Subject: [PATCH 47/92] Use boolean type for lookup_field.required field --- include/mongoose_mam.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mongoose_mam.hrl b/include/mongoose_mam.hrl index 1279086313..b306c31466 100644 --- a/include/mongoose_mam.hrl +++ b/include/mongoose_mam.hrl @@ -1,2 +1,2 @@ -record(db_mapping, {column :: atom(), param :: atom(), format :: atom()}). --record(lookup_field, {op :: atom(), column :: atom(), param :: atom(), required :: atom(), value_maker :: atom()}). +-record(lookup_field, {op :: atom(), column :: atom(), param :: atom(), required = false :: boolean(), value_maker :: atom()}). From 2acfb9e41d814c2cc7b6dc8d0d358a051ba46d03 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 9 Dec 2020 21:52:45 +0100 Subject: [PATCH 48/92] ADd specs for decoder/encoder --- src/mam/mam_decoder.erl | 11 +++++++++++ src/mam/mam_encoder.erl | 13 +++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/mam/mam_decoder.erl b/src/mam/mam_decoder.erl index 87055693f7..4183ba6860 100644 --- a/src/mam/mam_decoder.erl +++ b/src/mam/mam_decoder.erl @@ -4,24 +4,35 @@ -export([decode_muc_gdpr_row/2]). -export([decode_retraction_info/2]). +-type ext_mess_id() :: non_neg_integer() | binary(). -type env_vars() :: mod_mam_rdbms_arch:env_vars(). +-type db_row() :: {ext_mess_id(), ExtSrcJID :: binary(), ExtData :: binary()}. +-type db_muc_row() :: {ext_mess_id(), Nick :: binary(), ExtData :: binary()}. +-type db_muc_gdpr_row() :: {ext_mess_id(), ExtData :: binary()}. +-type decoded_muc_gdpr_row() :: {ext_mess_id(), exml:element()}. +-type retraction_info() :: #{packet := exml:element(), message_id := mod_mam:message_id()}. +-spec decode_row(db_row(), env_vars()) -> mod_mam:message_row(). decode_row({ExtMessID, ExtSrcJID, ExtData}, Env) -> MessID = mongoose_rdbms:result_to_integer(ExtMessID), SrcJID = decode_jid(ExtSrcJID, Env), Packet = decode_packet(ExtData, Env), {MessID, SrcJID, Packet}. +-spec decode_muc_row(db_muc_row(), env_vars()) -> mod_mam:message_row(). decode_muc_row({ExtMessID, Nick, ExtData}, Env = #{archive_jid := RoomJID}) -> MessID = mongoose_rdbms:result_to_integer(ExtMessID), SrcJID = jid:replace_resource(RoomJID, Nick), Packet = decode_packet(ExtData, Env), {MessID, SrcJID, Packet}. +-spec decode_muc_gdpr_row(db_muc_gdpr_row(), env_vars()) -> decoded_muc_gdpr_row(). decode_muc_gdpr_row({ExtMessID, ExtData}, Env) -> Packet = decode_packet(ExtData, Env), {ExtMessID, Packet}. +-spec decode_retraction_info(env_vars(), [] | [{mod_mam:message_id(), binary()}]) -> + skip | retraction_info(). decode_retraction_info(_Env, []) -> skip; decode_retraction_info(Env, [{ResMessID, Data}]) -> Packet = decode_packet(Data, Env), diff --git a/src/mam/mam_encoder.erl b/src/mam/mam_encoder.erl index aa0a3196cc..c7b58b616f 100644 --- a/src/mam/mam_encoder.erl +++ b/src/mam/mam_encoder.erl @@ -5,14 +5,17 @@ -export([encode_packet/2]). -export([extend_lookup_params/2]). --type value_type() :: int | maybe_string | direction | bare_jid | jid | jid_resource | xml | search. --type env_vars() :: mod_mam_rdbms_arch:env_vars(). - -include("mongoose.hrl"). -include("jlib.hrl"). -include_lib("exml/include/exml.hrl"). -include("mongoose_mam.hrl"). +-type value_type() :: int | maybe_string | direction | bare_jid | jid | jid_resource | xml | search. +-type env_vars() :: mod_mam_rdbms_arch:env_vars(). +-type db_mapping() :: #db_mapping{}. +-type encoded_field_value() :: term(). + +-spec extend_lookup_params(mam_iq:lookup_params(), env_vars()) -> mam_iq:lookup_params(). extend_lookup_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, borders := Borders, search_text := SearchText} = Params, Env) -> Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText), @@ -21,6 +24,8 @@ extend_lookup_params(#{start_ts := Start, end_ts := End, with_jid := WithJID, remote_bare_jid => maybe_encode_bare_jid(WithJID, Env), remote_resource => jid_to_non_empty_resource(WithJID)}. +-spec encode_message(mod_mam:archive_message_params(), env_vars(), list(db_mapping())) -> + [encoded_field_value()]. encode_message(Params, Env, Mappings) -> [encode_value_using_mapping(Params, Env, Mapping) || Mapping <- Mappings]. @@ -28,7 +33,7 @@ encode_value_using_mapping(Params, Env, #db_mapping{param = Param, format = Form Value = maps:get(Param, Params), encode_value(Format, Value, Env). --spec encode_value(value_type(), term(), env_vars()) -> term(). +-spec encode_value(value_type(), term(), env_vars()) -> encoded_field_value(). encode_value(int, Value, _Env) when is_integer(Value) -> Value; encode_value(maybe_string, none, _Env) -> From 4a3158ae104bba804a86eef761a8473003f92b8a Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 9 Dec 2020 22:00:57 +0100 Subject: [PATCH 49/92] Remove mod_mam_utils:get_retract_id/3 --- src/mam/mod_mam_utils.erl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/mam/mod_mam_utils.erl b/src/mam/mod_mam_utils.erl index ef38fa3184..44dff40068 100644 --- a/src/mam/mod_mam_utils.erl +++ b/src/mam/mod_mam_utils.erl @@ -31,7 +31,6 @@ is_archivable_message/4, has_message_retraction/2, get_retract_id/2, - get_retract_id/3, get_origin_id/1, tombstone/2, wrap_message/6, @@ -358,9 +357,6 @@ has_chat_marker(Packet) -> _ -> false end. -get_retract_id(Module, Host, Packet) -> - get_retract_id(has_message_retraction(Module, Host), Packet). - get_retract_id(true = _Enabled, Packet) -> get_retract_id(Packet); get_retract_id(false, Packet) -> From fda060803457e2da1142f8c73359b18ff51b5b83 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 10 Dec 2020 10:18:53 +0100 Subject: [PATCH 50/92] Apply code review comments --- src/mam/mod_mam_muc_rdbms_arch.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index 3abb1f640d..e40f7b2de0 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -241,9 +241,9 @@ retract_message(Host, #{archive_id := ArcID, sender_id := SenderID, packet := Packet}, Env) -> case get_retract_id(Packet, Env) of none -> ok; - OriginID -> - Info = get_retraction_info(Host, ArcID, SenderID, OriginID, Env), - make_tombstone(Host, ArcID, OriginID, Info, Env) + OriginIDToRetract -> + Info = get_retraction_info(Host, ArcID, SenderID, OriginIDToRetract, Env), + make_tombstone(Host, ArcID, OriginIDToRetract, Info, Env) end. get_retraction_info(Host, ArcID, SenderID, OriginID, Env) -> @@ -301,7 +301,7 @@ extract_gdpr_messages(Host, SenderID) -> %% Lookup logic -spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> {ok, mod_mam:lookup_result()}. -lookup_messages({error, _Reason}=Result, _Host, _Params) -> +lookup_messages({error, _Reason} = Result, _Host, _Params) -> Result; lookup_messages(_Result, Host, Params = #{owner_jid := ArcJID}) -> Env = env_vars(Host, ArcJID), From 80927310f3e5ac8dbc8f9f206d08e787ed19c809 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 14 Dec 2020 12:03:54 +0100 Subject: [PATCH 51/92] Add mongoose_rdbms:execute_successfully Address review comments --- src/mam/mam_decoder.erl | 1 - src/mam/mam_lookup.erl | 2 +- src/mam/mam_lookup_sql.erl | 21 ++----------------- src/mam/mod_mam_muc_rdbms_arch.erl | 13 ++++++------ src/mam/mod_mam_rdbms_arch.erl | 11 +++++----- src/mam/mod_mam_utils.erl | 7 +------ src/rdbms/mongoose_rdbms.erl | 33 ++++++++++++++++++++++++++++++ 7 files changed, 48 insertions(+), 40 deletions(-) diff --git a/src/mam/mam_decoder.erl b/src/mam/mam_decoder.erl index 4183ba6860..77b508041d 100644 --- a/src/mam/mam_decoder.erl +++ b/src/mam/mam_decoder.erl @@ -50,5 +50,4 @@ decode_packet(ExtBin, Env = #{db_message_codec := Codec}) -> -spec unescape_binary(binary(), env_vars()) -> binary(). unescape_binary(Bin, #{host := Host}) -> - %% Funny, rdbms ignores this Host variable mongoose_rdbms:unescape_binary(Host, Bin). diff --git a/src/mam/mam_lookup.erl b/src/mam/mam_lookup.erl index 6137f26bec..f25db76fa7 100644 --- a/src/mam/mam_lookup.erl +++ b/src/mam/mam_lookup.erl @@ -31,7 +31,7 @@ decode_row(Row, #{decode_row_fn := DecodeF} = Env) -> %% Private logic below -%% There is no optimizations for these queries yet: +%% There are no optimizations for these queries yet: %% - #rsm_in{direction = aft, id = ID} %% - #rsm_in{direction = before, id = ID} -spec opt_count_type(RSM :: maybe_rsm()) -> opt_count_type(). diff --git a/src/mam/mam_lookup_sql.erl b/src/mam/mam_lookup_sql.erl index e7e39bf631..670a49a98b 100644 --- a/src/mam/mam_lookup_sql.erl +++ b/src/mam/mam_lookup_sql.erl @@ -61,22 +61,7 @@ lookup_query(QueryType, Env, Filters, Order, OffsetLimit) -> ok end, Args = filters_to_args(Filters, OffsetLimit), - try mongoose_rdbms:execute(Host, StmtName, Args) of - {selected, Rs} when is_list(Rs) -> - {selected, Rs}; - Error -> - What = #{what => mam_lookup_failed, statement => StmtName, - sql_query => lookup_sql_binary(QueryType, Table, Env, Filters, Order, OffsetLimit), - reason => Error, host => Host}, - ?LOG_ERROR(What), - error(What) - catch error:Error:Stacktrace -> - What = #{what => mam_lookup_failed, statement => StmtName, - sql_query => lookup_sql_binary(QueryType, Table, Env, Filters, Order, OffsetLimit), - stacktrace => Stacktrace, reason => Error, host => Host}, - ?LOG_ERROR(What), - erlang:raise(error, Error, Stacktrace) - end. + mongoose_rdbms:execute_successfully(Host, StmtName, Args). lookup_sql_binary(QueryType, Table, Env, Filters, Order, OffsetLimit) -> iolist_to_binary(lookup_sql(QueryType, Table, Env, Filters, Order, OffsetLimit)). @@ -137,13 +122,11 @@ offset_limit_to_id(all) -> "all". filters_to_sql(Filters) -> SQLs = [filter_to_sql(Filter) || Filter <- Filters], - case skip_undefined(SQLs) of + case SQLs of [] -> ""; Defined -> [" WHERE ", rdbms_queries:join(Defined, " AND ")] end. -skip_undefined(List) -> [X || X <- List, X =/= undefined]. - -spec filter_to_sql(mam_filter:filter_field()) -> sql_part(). filter_to_sql({Op, Column, _Value}) -> filter_to_sql(atom_to_list(Column), Op). diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index e40f7b2de0..a53a7cb6c9 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -227,7 +227,7 @@ archive_message(_Result, Host, Params0 = #{local_jid := ArcJID}) -> do_archive_message(Host, Params, Env) -> Row = mam_encoder:encode_message(Params, Env, db_mappings()), - {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_muc_message, Row). + {updated, 1} = mongoose_rdbms:execute_successfully(Host, insert_mam_muc_message, Row). %% Retraction logic %% Called after inserting a new message @@ -261,13 +261,12 @@ make_tombstone(Host, ArcID, OriginID, #{packet := Packet, message_id := MessID}, execute_make_tombstone(Host, TombstoneData, ArcID, MessID). execute_select_messages_to_retract(Host, ArcID, SenderID, OriginID) -> - mod_mam_utils:success_sql_execute(Host, mam_muc_select_messages_to_retract, + mongoose_rdbms:execute_successfully(Host, mam_muc_select_messages_to_retract, [ArcID, SenderID, OriginID]). execute_make_tombstone(Host, TombstoneData, ArcID, MessID) -> - {updated, _} = - mod_mam_utils:success_sql_execute(Host, mam_muc_make_tombstone, - [TombstoneData, ArcID, MessID]). + mongoose_rdbms:execute_successfully(Host, mam_muc_make_tombstone, + [TombstoneData, ArcID, MessID]). %% Insert logic -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). @@ -292,11 +291,11 @@ remove_archive(Acc, Host, ArcID, _ArcJID) -> Acc. remove_archive(Host, ArcID) -> - {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_muc_archive_remove, [ArcID]). + mongoose_rdbms:execute_successfully(Host, mam_muc_archive_remove, [ArcID]). %% GDPR logic extract_gdpr_messages(Host, SenderID) -> - mod_mam_utils:success_sql_execute(Host, mam_muc_extract_gdpr_messages, [SenderID]). + mongoose_rdbms:execute_successfully(Host, mam_muc_extract_gdpr_messages, [SenderID]). %% Lookup logic -spec lookup_messages(Result :: any(), Host :: jid:server(), Params :: map()) -> diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index cc77fb1bc0..5ca21e954a 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -246,7 +246,7 @@ archive_message(_Result, Host, Params = #{local_jid := ArcJID}) -> do_archive_message(Host, Params, Env) -> Row = mam_encoder:encode_message(Params, Env, db_mappings()), - {updated, 1} = mod_mam_utils:success_sql_execute(Host, insert_mam_message, Row). + {updated, 1} = mongoose_rdbms:execute_successfully(Host, insert_mam_message, Row). %% Retraction logic %% Called after inserting a new message @@ -285,13 +285,12 @@ make_tombstone(Host, ArcID, OriginID, #{packet := Packet, message_id := MessID}, execute_make_tombstone(Host, TombstoneData, ArcID, MessID). execute_select_messages_to_retract(Host, ArcID, BareRemJID, OriginID, Dir) -> - mod_mam_utils:success_sql_execute(Host, mam_select_messages_to_retract, + mongoose_rdbms:execute_successfully(Host, mam_select_messages_to_retract, [ArcID, BareRemJID, OriginID, Dir]). execute_make_tombstone(Host, TombstoneData, ArcID, MessID) -> - {updated, _} = - mod_mam_utils:success_sql_execute(Host, mam_make_tombstone, - [TombstoneData, ArcID, MessID]). + mongoose_rdbms:execute_successfully(Host, mam_make_tombstone, + [TombstoneData, ArcID, MessID]). %% Insert logic -spec prepare_message(jid:server(), mod_mam:archive_message_params()) -> list(). @@ -316,7 +315,7 @@ remove_archive(Acc, Host, ArcID, _ArcJID) -> Acc. remove_archive(Host, ArcID) -> - {updated, _} = mod_mam_utils:success_sql_execute(Host, mam_archive_remove, [ArcID]). + mongoose_rdbms:execute_successfully(Host, mam_archive_remove, [ArcID]). %% GDPR logic extract_gdpr_messages(Env, ArcID) -> diff --git a/src/mam/mod_mam_utils.erl b/src/mam/mod_mam_utils.erl index 44dff40068..0ce8b8ff5b 100644 --- a/src/mam/mod_mam_utils.erl +++ b/src/mam/mod_mam_utils.erl @@ -69,7 +69,7 @@ expand_minified_jid/2]). %% SQL --export([success_sql_query/2, success_sql_execute/3]). +-export([success_sql_query/2]). %% Other -export([maybe_integer/2, @@ -1099,11 +1099,6 @@ success_sql_query(HostOrConn, Query) -> Result = mongoose_rdbms:sql_query(HostOrConn, Query), error_on_sql_error(HostOrConn, Query, Result). --spec success_sql_execute(atom() | jid:server(), atom(), [term()]) -> any(). -success_sql_execute(HostOrConn, Name, Params) -> - Result = mongoose_rdbms:execute(HostOrConn, Name, Params), - error_on_sql_error(HostOrConn, Name, Result). - error_on_sql_error(HostOrConn, Query, {error, Reason}) -> ?LOG_ERROR(#{what => mam_sql_error, host => HostOrConn, query => Query, reason => Reason}), diff --git a/src/rdbms/mongoose_rdbms.erl b/src/rdbms/mongoose_rdbms.erl index 300e77a477..911e28a75c 100644 --- a/src/rdbms/mongoose_rdbms.erl +++ b/src/rdbms/mongoose_rdbms.erl @@ -78,6 +78,7 @@ -export([prepare/4, prepared/1, execute/3, + execute_successfully/3, sql_query/2, sql_query_t/1, sql_transaction/2, @@ -191,6 +192,38 @@ prepared(Name) -> execute(Host, Name, Parameters) when is_atom(Name), is_list(Parameters) -> sql_call(Host, {sql_execute, Name, Parameters}). +%% Same as execute/3, but would fail loudly on any error. +-spec execute_successfully(Host :: server(), Name :: atom(), Parameters :: [term()]) -> + query_result(). +execute_successfully(Host, Name, Parameters) -> + try execute(Host, Name, Parameters) of + {selected, _} = Result -> + Result; + {updated, _} = Result -> + Result; + Other -> + Log = #{what => sql_execute_failed, host => Host,statement_name => Name, + statement_query => query_name_to_string(Name), + statement_params => Parameters, reason => Other}, + ?LOG_ERROR(Log), + error(Log) + catch error:Reason:Stacktrace -> + Log = #{what => sql_execute_failed, host => Host, statement_name => Name, + statement_query => query_name_to_string(Name), + statement_params => Parameters, + reason => Reason, stacktrace => Stacktrace}, + ?LOG_ERROR(Log), + erlang:raise(error, Reason, Stacktrace) + end. + +query_name_to_string(Name) -> + case ets:lookup(prepared_statements, Name) of + [] -> + not_found; + [{_, _Table, _Fields, Statement}] -> + Statement + end. + -spec sql_query(Host :: server(), Query :: any()) -> query_result(). sql_query(Host, Query) -> sql_call(Host, {sql_query, Query}). From 58afe4da6c61849459a289f80521e3ec8b878bc6 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 14 Dec 2020 12:04:35 +0100 Subject: [PATCH 52/92] Pass binary into mongoose_rdbms:prepare/4 from mam modules --- src/mam/mod_mam_muc_rdbms_arch.erl | 27 ++++++++++++--------------- src/mam/mod_mam_rdbms_arch.erl | 23 ++++++++++------------- src/mam/mod_mam_utils.erl | 4 ++-- 3 files changed, 24 insertions(+), 30 deletions(-) diff --git a/src/mam/mod_mam_muc_rdbms_arch.erl b/src/mam/mod_mam_muc_rdbms_arch.erl index a53a7cb6c9..d4ba4ba68e 100644 --- a/src/mam/mod_mam_muc_rdbms_arch.erl +++ b/src/mam/mod_mam_muc_rdbms_arch.erl @@ -110,22 +110,22 @@ stop_hooks(Host) -> register_prepared_queries() -> prepare_insert(insert_mam_muc_message, 1), mongoose_rdbms:prepare(mam_muc_archive_remove, mam_muc_message, [room_id], - [<<"DELETE FROM mam_muc_message " - "WHERE room_id = ?">>]), + <<"DELETE FROM mam_muc_message " + "WHERE room_id = ?">>), mongoose_rdbms:prepare(mam_muc_make_tombstone, mam_muc_message, [message, room_id, id], - [<<"UPDATE mam_muc_message SET message = ?, search_body = '' " - "WHERE room_id = ? AND id = ?">>]), + <<"UPDATE mam_muc_message SET message = ?, search_body = '' " + "WHERE room_id = ? AND id = ?">>), {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(1), mongoose_rdbms:prepare(mam_muc_select_messages_to_retract, mam_muc_message, [room_id, sender_id, origin_id], - [<<"SELECT ", LimitMSSQL/binary, - " id, message FROM mam_muc_message" - " WHERE room_id = ? AND sender_id = ? " - " AND origin_id = ?" - " ORDER BY id DESC ", LimitSQL/binary>>]), + <<"SELECT ", LimitMSSQL/binary, + " id, message FROM mam_muc_message" + " WHERE room_id = ? AND sender_id = ? " + " AND origin_id = ?" + " ORDER BY id DESC ", LimitSQL/binary>>), mongoose_rdbms:prepare(mam_muc_extract_gdpr_messages, mam_muc_message, [sender_id], - [<<"SELECT id, message FROM mam_muc_message " - " WHERE sender_id = ? ORDER BY id">>]). + <<"SELECT id, message FROM mam_muc_message " + " WHERE sender_id = ? ORDER BY id">>). %% ---------------------------------------------------------------------- %% Declarative logic @@ -287,12 +287,9 @@ prepare_insert(Name, NumRows) -> ArcID :: mod_mam:archive_id(), ArcJID :: jid:jid()) -> mongoose_acc:t(). remove_archive(Acc, Host, ArcID, _ArcJID) -> - remove_archive(Host, ArcID), + mongoose_rdbms:execute_successfully(Host, mam_muc_archive_remove, [ArcID]), Acc. -remove_archive(Host, ArcID) -> - mongoose_rdbms:execute_successfully(Host, mam_muc_archive_remove, [ArcID]). - %% GDPR logic extract_gdpr_messages(Host, SenderID) -> mongoose_rdbms:execute_successfully(Host, mam_muc_extract_gdpr_messages, [SenderID]). diff --git a/src/mam/mod_mam_rdbms_arch.erl b/src/mam/mod_mam_rdbms_arch.erl index 5ca21e954a..496727a3d0 100644 --- a/src/mam/mod_mam_rdbms_arch.erl +++ b/src/mam/mod_mam_rdbms_arch.erl @@ -126,19 +126,19 @@ stop_hooks(Host) -> register_prepared_queries() -> prepare_insert(insert_mam_message, 1), mongoose_rdbms:prepare(mam_archive_remove, mam_message, [user_id], - [<<"DELETE FROM mam_message " - "WHERE user_id = ?">>]), + <<"DELETE FROM mam_message " + "WHERE user_id = ?">>), mongoose_rdbms:prepare(mam_make_tombstone, mam_message, [message, user_id, id], - [<<"UPDATE mam_message SET message = ?, search_body = '' " - "WHERE user_id = ? AND id = ?">>]), + <<"UPDATE mam_message SET message = ?, search_body = '' " + "WHERE user_id = ? AND id = ?">>), {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(1), mongoose_rdbms:prepare(mam_select_messages_to_retract, mam_message, [user_id, remote_bare_jid, origin_id, direction], - [<<"SELECT ", LimitMSSQL/binary, - " id, message FROM mam_message" - " WHERE user_id = ? AND remote_bare_jid = ? " - " AND origin_id = ? AND direction = ?" - " ORDER BY id DESC ", LimitSQL/binary>>]). + <<"SELECT ", LimitMSSQL/binary, + " id, message FROM mam_message" + " WHERE user_id = ? AND remote_bare_jid = ? " + " AND origin_id = ? AND direction = ?" + " ORDER BY id DESC ", LimitSQL/binary>>). %% ---------------------------------------------------------------------- %% Declarative logic @@ -311,12 +311,9 @@ prepare_insert(Name, NumRows) -> ArcID :: mod_mam:archive_id(), RoomJID :: jid:jid()) -> mongoose_acc:t(). remove_archive(Acc, Host, ArcID, _ArcJID) -> - remove_archive(Host, ArcID), + mongoose_rdbms:execute_successfully(Host, mam_archive_remove, [ArcID]), Acc. -remove_archive(Host, ArcID) -> - mongoose_rdbms:execute_successfully(Host, mam_archive_remove, [ArcID]). - %% GDPR logic extract_gdpr_messages(Env, ArcID) -> Filters = [{equal, user_id, ArcID}], diff --git a/src/mam/mod_mam_utils.erl b/src/mam/mod_mam_utils.erl index 0ce8b8ff5b..aaa1f0deb0 100644 --- a/src/mam/mod_mam_utils.erl +++ b/src/mam/mod_mam_utils.erl @@ -359,7 +359,7 @@ has_chat_marker(Packet) -> get_retract_id(true = _Enabled, Packet) -> get_retract_id(Packet); -get_retract_id(false, Packet) -> +get_retract_id(false, _Packet) -> none. get_retract_id(Packet) -> @@ -770,7 +770,7 @@ packet_to_search_body(Module, Host, Packet) -> packet_to_search_body(true, Packet) -> BodyValue = exml_query:path(Packet, [{element, <<"body">>}, cdata], <<>>), mod_mam_utils:normalize_search_text(BodyValue, <<" ">>); -packet_to_search_body(false, Packet) -> +packet_to_search_body(false, _Packet) -> <<>>. -spec has_full_text_search(Module :: mod_mam | mod_mam_muc, Host :: jid:server()) -> boolean(). From a8b0196784daab194fe1ca7594dde5620a696f1a Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 14 Dec 2020 12:13:33 +0100 Subject: [PATCH 53/92] Store statement as binary in prepared_statements ETS table So, we can more easily print it Also, ets-lookup logic would run a bit faster for big iolists --- src/rdbms/mongoose_rdbms.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rdbms/mongoose_rdbms.erl b/src/rdbms/mongoose_rdbms.erl index 911e28a75c..d452997a2f 100644 --- a/src/rdbms/mongoose_rdbms.erl +++ b/src/rdbms/mongoose_rdbms.erl @@ -178,7 +178,8 @@ prepare(Name, Table, [Field | _] = Fields, Statement) when is_atom(Field) -> prepare(Name, Table, [atom_to_binary(F, utf8) || F <- Fields], Statement); prepare(Name, Table, Fields, Statement) when is_atom(Name), is_binary(Table) -> true = lists:all(fun is_binary/1, Fields), - case ets:insert_new(prepared_statements, {Name, Table, Fields, Statement}) of + Tuple = {Name, Table, Fields, iolist_to_binary(Statement)}, + case ets:insert_new(prepared_statements, Tuple) of true -> {ok, Name}; false -> {error, already_exists} end. From c8fc5901b59bf757707a1c6d22456ee3069f1569 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 10 Dec 2020 11:03:48 +0100 Subject: [PATCH 54/92] Use prepared queries in mod_mam_rdbms_user --- src/mam/mod_mam_rdbms_user.erl | 87 ++++++++++------------------------ 1 file changed, 26 insertions(+), 61 deletions(-) diff --git a/src/mam/mod_mam_rdbms_user.erl b/src/mam/mod_mam_rdbms_user.erl index 6f5f7c437f..816cf91e57 100644 --- a/src/mam/mod_mam_rdbms_user.erl +++ b/src/mam/mod_mam_rdbms_user.erl @@ -17,9 +17,6 @@ -export([archive_id/3, remove_archive/4]). -%% gdpr functions --export([get_archive_id/2]). - %% For debugging ONLY -export([create_user_archive/3]). @@ -67,6 +64,7 @@ stop(Host) -> -spec start_pm(jid:server(), _) -> 'ok'. start_pm(Host, _Opts) -> + prepare_queries(), ejabberd_hooks:add(mam_archive_id, Host, ?MODULE, archive_id, 50), case gen_mod:get_module_opt(Host, ?MODULE, auto_remove, false) of true -> @@ -106,7 +104,6 @@ start_muc(Host, _Opts) -> end, ok. - -spec stop_muc(jid:server()) -> 'ok'. stop_muc(Host) -> ejabberd_hooks:delete(mam_muc_archive_id, Host, ?MODULE, archive_id, 50), @@ -119,30 +116,26 @@ stop_muc(Host) -> end, ok. +%% Preparing queries +prepare_queries() -> + mongoose_rdbms:prepare(mam_user_remove, mam_server_user, [server, user_name], + <<"DELETE FROM mam_server_user WHERE server=? AND user_name=?">>), + mongoose_rdbms:prepare(mam_user_select, mam_server_user, [server, user_name], + <<"SELECT id FROM mam_server_user WHERE server=? AND user_name=?">>), + mongoose_rdbms:prepare(mam_user_insert, mam_server_user, [server, user_name], + <<"INSERT INTO mam_server_user (server, user_name) VALUES (?, ?)">>), + ok. %%==================================================================== %% API %%==================================================================== -spec archive_id(undefined | mod_mam:archive_id(), jid:server(), jid:jid()) -> mod_mam:archive_id(). -archive_id(undefined, Host, _ArcJID=#jid{lserver = Server, luser = UserName}) -> - query_archive_id(Host, Server, UserName); +archive_id(undefined, Host, _ArcJID=#jid{lserver = LServer, luser = LUser}) -> + query_archive_id(Host, LServer, LUser); archive_id(ArcID, _Host, _ArcJID) -> ArcID. --spec get_archive_id(jid:server(), jid:user()) -> undefined | mod_mam:archive_id(). -get_archive_id(Host, User) -> - #jid{lserver = Server, luser = UserName} = jid:make(User, Host, <<"">>), - SServer = mongoose_rdbms:escape_string(Server), - SUserName = mongoose_rdbms:escape_string(UserName), - DbType = mongoose_rdbms_type:get(), - case do_query_archive_id(DbType, Host, SServer, SUserName) of - {selected, [{IdBin}]} -> - mongoose_rdbms:result_to_integer(IdBin); - {selected, []} -> - undefined - end. - -spec remove_archive(Acc :: map(), Host :: jid:server(), ArchiveID :: mod_mam:archive_id(), ArchiveJID :: jid:jid()) -> map(). @@ -150,55 +143,38 @@ remove_archive(Acc, Host, ArcID, ArcJID) -> remove_archive(Host, ArcID, ArcJID), Acc. -remove_archive(Host, _ArcID, _ArcJID=#jid{lserver = Server, luser = UserName}) -> - SUserName = mongoose_rdbms:escape_string(UserName), - SServer = mongoose_rdbms:escape_string(Server), +remove_archive(Host, _ArcID, _ArcJID=#jid{lserver = LServer, luser = LUser}) -> {updated, _} = - mongoose_rdbms:sql_query( - Host, - ["DELETE FROM mam_server_user " - "WHERE server = ", mongoose_rdbms:use_escaped_string(SServer), - " AND user_name = ", mongoose_rdbms:use_escaped_string(SUserName)]). + mongoose_rdbms:execute(Host, mam_user_remove, [LUser, LServer]). + %%==================================================================== %% Internal functions %%==================================================================== -spec query_archive_id(jid:server(), jid:lserver(), jid:user()) -> integer(). -query_archive_id(Host, Server, UserName) -> +query_archive_id(Host, LServer, LUser) -> Tries = 5, - query_archive_id(Host, Server, UserName, Tries). + query_archive_id(Host, LServer, LUser, Tries). -query_archive_id(Host, Server, UserName, 0) -> +query_archive_id(Host, LServer, LUser, 0) -> ?LOG_ERROR(#{what => query_archive_id_failed, - host => Host, server => Server, user => UserName}), + host => Host, server => LServer, user => LUser}), error(query_archive_id_failed); -query_archive_id(Host, Server, UserName, Tries) when Tries > 0 -> - SServer = mongoose_rdbms:escape_string(Server), - SUserName = mongoose_rdbms:escape_string(UserName), - DbType = mongoose_rdbms_type:get(), - Result = do_query_archive_id(DbType, Host, SServer, SUserName), - +query_archive_id(Host, LServer, LUser, Tries) when Tries > 0 -> + Result = mongoose_rdbms:execute(Host, mam_user_select, [LServer, LUser]), case Result of {selected, [{IdBin}]} -> mongoose_rdbms:result_to_integer(IdBin); {selected, []} -> %% The user is not found - create_user_archive(Host, Server, UserName), - query_archive_id(Host, Server, UserName, Tries - 1) + create_user_archive(Host, LServer, LUser), + query_archive_id(Host, LServer, LUser, Tries - 1) end. -spec create_user_archive(jid:server(), jid:lserver(), jid:user()) -> ok. -create_user_archive(Host, Server, UserName) -> - SServer = mongoose_rdbms:escape_string(Server), - SUserName = mongoose_rdbms:escape_string(UserName), - Res = - mongoose_rdbms:sql_query( - Host, - ["INSERT INTO mam_server_user " - "(server, user_name) VALUES (", - mongoose_rdbms:use_escaped_string(SServer), ", ", - mongoose_rdbms:use_escaped_string(SUserName), ")"]), +create_user_archive(Host, LServer, LUser) -> + Res = mongoose_rdbms:execute(Host, mam_user_insert, [LServer, LUser]), case Res of {updated, 1} -> ok; @@ -210,17 +186,6 @@ create_user_archive(Host, Server, UserName) -> %% - {error, "[FreeTDS][SQL Server]Violation of UNIQUE KEY constraint" ++ _} %% Let's ignore the errors and just retry in query_archive_id ?LOG_WARNING(#{what => create_user_archive_failed, reason => Res, - user => UserName, host => Host, server => Server}), + user => LUser, host => Host, server => LServer}), ok end. - -do_query_archive_id(mssql, Host, SServer, SUserName) -> - rdbms_queries_mssql:query_archive_id(Host, SServer, SUserName); -do_query_archive_id(_, Host, SServer, SUserName) -> - mongoose_rdbms:sql_query( - Host, - ["SELECT id " - "FROM mam_server_user " - "WHERE server = ", mongoose_rdbms:use_escaped_string(SServer), - " AND user_name = ", mongoose_rdbms:use_escaped_string(SUserName), " " - "LIMIT 1"]). From 19c43bab029d7703238d0cb7e0b04e83bfa0fcc4 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 10 Dec 2020 12:06:50 +0100 Subject: [PATCH 55/92] Convert mod_mam_rdbms_prefs to prepared queries --- src/mam/mod_mam_rdbms_prefs.erl | 208 +++++++++++++------------------- 1 file changed, 83 insertions(+), 125 deletions(-) diff --git a/src/mam/mod_mam_rdbms_prefs.erl b/src/mam/mod_mam_rdbms_prefs.erl index 9223fa6a73..6c924ee667 100644 --- a/src/mam/mod_mam_rdbms_prefs.erl +++ b/src/mam/mod_mam_rdbms_prefs.erl @@ -36,6 +36,7 @@ -spec start(jid:server(), _) -> 'ok'. start(Host, Opts) -> + prepare_queries(Host), case gen_mod:get_module_opt(Host, ?MODULE, pm, false) of true -> start_pm(Host, Opts); @@ -66,6 +67,40 @@ stop(Host) -> end. +%% Prepared queries +prepare_queries(Host) -> + OrdBy = order_by_remote_jid_in_delete(Host), + mongoose_rdbms:prepare(mam_prefs_delete, mam_config, [user_id], + <<"DELETE FROM mam_config WHERE user_id=?", OrdBy/binary>>), + mongoose_rdbms:prepare(mam_prefs_insert, mam_config, [user_id, remote_jid, behaviour], + <<"INSERT INTO mam_config(user_id, remote_jid, behaviour) " + "VALUES (?, ?, ?)">>), + mongoose_rdbms:prepare(mam_prefs_select, mam_config, [user_id], + <<"SELECT remote_jid, behaviour " + "FROM mam_config WHERE user_id=?">>), + mongoose_rdbms:prepare(mam_prefs_select_behaviour, mam_config, + [user_id, remote_jid], + <<"SELECT remote_jid, behaviour " + "FROM mam_config " + "WHERE user_id=? " + "AND (remote_jid='' OR remote_jid=?)">>), + mongoose_rdbms:prepare(mam_prefs_select_behaviour2, mam_config, + [user_id, remote_jid, remote_jid], + <<"SELECT remote_jid, behaviour " + "FROM mam_config " + "WHERE user_id=? " + "AND (remote_jid='' OR remote_jid=? OR remote_jid=?)">>), + ok. + +order_by_remote_jid_in_delete(Host) -> + case mongoose_rdbms:db_engine(Host) of + mysql -> + <<" ORDER BY remote_jid">>; + _ -> + <<"">> + end. + + %% ---------------------------------------------------------------------- %% Add hooks for mod_mam @@ -119,12 +154,7 @@ get_behaviour(DefaultBehaviour, Host, UserID, _LocJID, RemJID) RemLJID = jid:to_lower(RemJID), BRemLBareJID = jid:to_binary(jid:to_bare(RemLJID)), BRemLJID = jid:to_binary(RemLJID), - SRemLBareJID = escape_string(BRemLBareJID), - SRemLJID = escape_string(BRemLJID), - SUserID = escape_integer(UserID), - %% CheckBare if resource is not empty - CheckBare = RemJID#jid.lresource =/= <<>>, - case query_behaviour(Host, SUserID, SRemLJID, SRemLBareJID, CheckBare) of + case query_behaviour(Host, UserID, BRemLJID, BRemLBareJID) of {selected, []} -> DefaultBehaviour; {selected, RemoteJid2Behaviour} -> @@ -160,79 +190,23 @@ set_prefs(_Result, Host, UserID, _ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> {error, Error} end. - -order_by_in_delete(Host) -> - case mongoose_rdbms:db_engine(Host) of - mysql -> - " ORDER BY remote_jid"; - _ -> - "" - end. - set_prefs1(Host, UserID, DefaultMode, AlwaysJIDs, NeverJIDs) -> - %% Lock keys in the same order to avoid deadlock - SUserID = escape_integer(UserID), - EscapedA = escape_string("A"), - EscapedN = escape_string("N"), - JidBehaviourA = [{JID, EscapedA} || JID <- AlwaysJIDs], - JidBehaviourN = [{JID, EscapedN} || JID <- NeverJIDs], - JidBehaviour = lists:keysort(1, JidBehaviourA ++ JidBehaviourN), - ValuesAN = [encode_config_row(SUserID, SBehaviour, escape_string(JID)) - || {JID, SBehaviour} <- JidBehaviour], - SDefaultMode = escape_string(encode_behaviour(DefaultMode)), - DefaultValue = encode_first_config_row(SUserID, SDefaultMode, escape_string("")), - Values = [DefaultValue|ValuesAN], - DelQuery = ["DELETE FROM mam_config WHERE user_id = ", - mongoose_rdbms:use_escaped_integer(SUserID), - order_by_in_delete(Host)], - InsQuery = ["INSERT INTO mam_config(user_id, behaviour, remote_jid) " - "VALUES ", Values], - - run_transaction_or_retry_on_abort(fun() -> - case sql_transaction_map(Host, [DelQuery, InsQuery]) of - {atomic, [{updated, _}, {updated, _}]} -> - {atomic, ok}; - Other -> - Other - end + Rows = prefs_to_rows(UserID, DefaultMode, AlwaysJIDs, NeverJIDs), + run_transaction_or_retry_on_abort(Host, fun() -> + {updated, _} = + mongoose_rdbms:execute(Host, mam_prefs_delete, [UserID]), + [mongoose_rdbms:execute(Host, mam_prefs_insert, Row) || Row <- Rows], + ok end, UserID, 5), ok. -%% Possible error with mysql -%% Reason "Deadlock found when trying to get lock; try restarting transaction" -%% triggered mod_mam_utils:error_on_sql_error -run_transaction_or_retry_on_abort(F, UserID, Retries) -> - Result = F(), - case Result of - {atomic, _} -> - Result; - {aborted, Reason} when Retries > 0 -> - ?LOG_WARNING(#{what => mam_transaction_aborted, - text => <<"Transaction aborted. Restart">>, - user_id => UserID, reason => Reason, retries => Retries}), - timer:sleep(100), - run_transaction_or_retry_on_abort(F, UserID, Retries-1); - _ -> - ?LOG_ERROR(#{what => mam_transaction_failed, - text => <<"Transaction failed. Do not restart">>, - user_id => UserID, reason => Result, retries => Retries}), - erlang:error({transaction_failed, #{user_id => UserID, result => Result}}) - end. - -spec get_prefs(mod_mam:preference(), _Host :: jid:server(), ArchiveID :: mod_mam:archive_id(), ArchiveJID :: jid:jid()) -> mod_mam:preference(). get_prefs({GlobalDefaultMode, _, _}, Host, UserID, _ArcJID) -> - SUserID = escape_integer(UserID), - {selected, Rows} = - mod_mam_utils:success_sql_query( - Host, - ["SELECT remote_jid, behaviour " - "FROM mam_config " - "WHERE user_id=", use_escaped_integer(SUserID)]), + {selected, Rows} = mongoose_rdbms:execute(Host, mam_prefs_select, [UserID]), decode_prefs_rows(Rows, GlobalDefaultMode, [], []). - -spec remove_archive(mongoose_acc:t(), jid:server(), mod_mam:archive_id(), jid:jid()) -> mongoose_acc:t(). remove_archive(Acc, Host, UserID, _ArcJID) -> @@ -240,77 +214,61 @@ remove_archive(Acc, Host, UserID, _ArcJID) -> Acc. remove_archive(Host, UserID) -> - SUserID = escape_integer(UserID), {updated, _} = - mod_mam_utils:success_sql_query( - Host, ["DELETE FROM mam_config WHERE user_id=", use_escaped_integer(SUserID)]). + mongoose_rdbms:execute(Host, mam_prefs_delete, [UserID]). -spec query_behaviour(jid:server(), - SUserID :: mongoose_rdbms:escaped_integer(), - SRemLJID :: mongoose_rdbms:escaped_string(), - SRemLBareJID :: mongoose_rdbms:escaped_string(), - CheckBare :: boolean() + UserID :: non_neg_integer(), + BRemLJID :: binary(), + BRemLBareJID :: binary() ) -> any(). -query_behaviour(Host, SUserID, SRemLJID, SRemLBareJID, CheckBare) -> - Result = - mod_mam_utils:success_sql_query( - Host, - ["SELECT remote_jid, behaviour " - "FROM mam_config " - "WHERE user_id=", use_escaped_integer(SUserID), " " - "AND (remote_jid='' OR remote_jid=", use_escaped_string(SRemLJID), - case CheckBare of - false -> - ""; - true -> - [" OR remote_jid=", use_escaped_string(SRemLBareJID)] - end, - ")"]), - ?LOG_DEBUG(#{what => mam_query_behaviour_result, - user_id => SUserID, result => Result}), - Result. +query_behaviour(Host, UserID, BRemLJID, BRemLJID) -> + mongoose_rdbms:execute(Host, mam_prefs_select_behaviour, + [UserID, BRemLJID]); %% check just bare jid +query_behaviour(Host, UserID, BRemLJID, BRemLBareJID) -> + mongoose_rdbms:execute(Host, mam_prefs_select_behaviour2, + [UserID, BRemLJID, BRemLBareJID]). %% ---------------------------------------------------------------------- %% Helpers --spec encode_behaviour(always | never | roster) -> string(). -encode_behaviour(roster) -> "R"; -encode_behaviour(always) -> "A"; -encode_behaviour(never) -> "N". +%% Possible error with mysql +%% Reason "Deadlock found when trying to get lock; try restarting transaction" +%% triggered mod_mam_utils:error_on_sql_error +run_transaction_or_retry_on_abort(Host, F, UserID, Retries) -> + Result = mongoose_rdbms:sql_transaction(Host, F), + case Result of + {atomic, _} -> + Result; + {aborted, Reason} when Retries > 0 -> + ?LOG_WARNING(#{what => mam_transaction_aborted, + text => <<"Transaction aborted. Restart">>, + user_id => UserID, reason => Reason, retries => Retries}), + timer:sleep(100), + run_transaction_or_retry_on_abort(Host, F, UserID, Retries-1); + _ -> + ?LOG_ERROR(#{what => mam_transaction_failed, + text => <<"Transaction failed. Do not restart">>, + user_id => UserID, reason => Result, retries => Retries}), + erlang:error({transaction_failed, #{user_id => UserID, result => Result}}) + end. +-spec encode_behaviour(always | never | roster) -> binary(). +encode_behaviour(roster) -> <<"R">>; +encode_behaviour(always) -> <<"A">>; +encode_behaviour(never) -> <<"N">>. -spec decode_behaviour(binary()) -> always | never | roster. decode_behaviour(<<"R">>) -> roster; decode_behaviour(<<"A">>) -> always; decode_behaviour(<<"N">>) -> never. --spec encode_first_config_row(SUserID :: mongoose_rdbms:escaped_integer(), - SBehaviour :: mongoose_rdbms:escaped_string(), - SJID :: mongoose_rdbms:escaped_string()) -> - mongoose_rdbms:sql_query_part(). -encode_first_config_row(SUserID, SBehaviour, SJID) -> - ["(", use_escaped_integer(SUserID), - ", ", use_escaped_string(SBehaviour), - ", ", use_escaped_string(SJID), ")"]. - - --spec encode_config_row(SUserID :: mongoose_rdbms:escaped_integer(), - SBehaviour :: mongoose_rdbms:escaped_string(), - SJID :: mongoose_rdbms:escaped_string()) -> - mongoose_rdbms:sql_query_part(). -encode_config_row(SUserID, SBehaviour, SJID) -> - [", (", use_escaped_integer(SUserID), - ", ", use_escaped_string(SBehaviour), - ", ", use_escaped_string(SJID), ")"]. - - --spec sql_transaction_map(jid:server(), [mongoose_rdbms:sql_query()]) -> any(). -sql_transaction_map(LServer, Queries) -> - AtomicF = fun() -> - [mod_mam_utils:success_sql_query(LServer, Query) || Query <- Queries] - end, - mongoose_rdbms:sql_transaction(LServer, AtomicF). - +prefs_to_rows(UserID, DefaultMode, AlwaysJIDs, NeverJIDs) -> + AlwaysRows = [[UserID, JID, encode_behaviour(always)] || JID <- AlwaysJIDs], + NeverRows = [[UserID, JID, encode_behaviour(never)] || JID <- NeverJIDs], + DefaultRow = [UserID, <<>>, encode_behaviour(DefaultMode)], + %% Lock keys in the same order to avoid deadlock + [DefaultRow|lists:sort(AlwaysRows ++ NeverRows)]. -spec decode_prefs_rows([{binary() | jid:jid(), binary()}], DefaultMode :: mod_mam:archive_behaviour(), From 692009c551a9ab8a49c12d9ee56f9bcca8cbd321 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 10 Dec 2020 12:19:03 +0100 Subject: [PATCH 56/92] Declare queries in INSERT, SELECT, DELETE order --- src/mam/mod_mam_rdbms_prefs.erl | 6 +++--- src/mam/mod_mam_rdbms_user.erl | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/mam/mod_mam_rdbms_prefs.erl b/src/mam/mod_mam_rdbms_prefs.erl index 6c924ee667..bb69116ec9 100644 --- a/src/mam/mod_mam_rdbms_prefs.erl +++ b/src/mam/mod_mam_rdbms_prefs.erl @@ -69,9 +69,6 @@ stop(Host) -> %% Prepared queries prepare_queries(Host) -> - OrdBy = order_by_remote_jid_in_delete(Host), - mongoose_rdbms:prepare(mam_prefs_delete, mam_config, [user_id], - <<"DELETE FROM mam_config WHERE user_id=?", OrdBy/binary>>), mongoose_rdbms:prepare(mam_prefs_insert, mam_config, [user_id, remote_jid, behaviour], <<"INSERT INTO mam_config(user_id, remote_jid, behaviour) " "VALUES (?, ?, ?)">>), @@ -90,6 +87,9 @@ prepare_queries(Host) -> "FROM mam_config " "WHERE user_id=? " "AND (remote_jid='' OR remote_jid=? OR remote_jid=?)">>), + OrdBy = order_by_remote_jid_in_delete(Host), + mongoose_rdbms:prepare(mam_prefs_delete, mam_config, [user_id], + <<"DELETE FROM mam_config WHERE user_id=?", OrdBy/binary>>), ok. order_by_remote_jid_in_delete(Host) -> diff --git a/src/mam/mod_mam_rdbms_user.erl b/src/mam/mod_mam_rdbms_user.erl index 816cf91e57..21a6f578c1 100644 --- a/src/mam/mod_mam_rdbms_user.erl +++ b/src/mam/mod_mam_rdbms_user.erl @@ -118,12 +118,12 @@ stop_muc(Host) -> %% Preparing queries prepare_queries() -> - mongoose_rdbms:prepare(mam_user_remove, mam_server_user, [server, user_name], - <<"DELETE FROM mam_server_user WHERE server=? AND user_name=?">>), - mongoose_rdbms:prepare(mam_user_select, mam_server_user, [server, user_name], - <<"SELECT id FROM mam_server_user WHERE server=? AND user_name=?">>), mongoose_rdbms:prepare(mam_user_insert, mam_server_user, [server, user_name], <<"INSERT INTO mam_server_user (server, user_name) VALUES (?, ?)">>), + mongoose_rdbms:prepare(mam_user_select, mam_server_user, [server, user_name], + <<"SELECT id FROM mam_server_user WHERE server=? AND user_name=?">>), + mongoose_rdbms:prepare(mam_user_remove, mam_server_user, [server, user_name], + <<"DELETE FROM mam_server_user WHERE server=? AND user_name=?">>), ok. %%==================================================================== From 7891095f5009054b6fa3285e208aa3b4233579b0 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 16 Dec 2020 11:34:06 +0100 Subject: [PATCH 57/92] Do not start mod_mam_rdbms_user for elasticsearch_and_cassandra_mnesia preset in big tests --- big_tests/tests/mam_SUITE.erl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/big_tests/tests/mam_SUITE.erl b/big_tests/tests/mam_SUITE.erl index 8e8e22c810..87a721af57 100644 --- a/big_tests/tests/mam_SUITE.erl +++ b/big_tests/tests/mam_SUITE.erl @@ -616,7 +616,7 @@ init_per_group(muc_rsm04, Config) -> init_per_group(Group, ConfigIn) -> C = configuration(Group), B = basic_group(Group), - case init_modules(C, B, ConfigIn) of + try init_modules(C, B, ConfigIn) of skip -> {skip, print_configuration_not_supported(C, B)}; Config0 -> @@ -624,7 +624,11 @@ init_per_group(Group, ConfigIn) -> [Group, C, B]), Config1 = do_init_per_group(C, Config0), [{basic_group, B}, {configuration, C} | init_state(C, B, Config1)] - end. + catch Class:Reason:Stacktrace -> + ct:pal("Failed to start configuration=~p basic_group=~p", + [C, B]), + erlang:raise(Class, Reason, Stacktrace) + end. backup_module_opts(Module) -> {{params_backup, Module}, rpc_apply(gen_mod, get_module_opts, [host(), mod_mam_muc])}. @@ -753,7 +757,11 @@ init_modules(rdbms_mnesia_cache, C, Config) when C =:= muc_all; Config; init_modules(BackendType, muc_light, Config) -> Config1 = init_modules_for_muc_light(BackendType, Config), - init_module(host(), mod_mam_rdbms_user, [muc, pm]), + case BackendType of + cassandra -> ok; + elasticsearch -> ok; + _ -> init_module(host(), mod_mam_rdbms_user, [muc, pm]) + end, Config1; init_modules(rdbms, C, Config) -> init_module(host(), mod_mam, addin_mam_options(C, Config)), From 3c3b0a7c9c79e5eac68e1b72d1e4fa66fdb4a821 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 16 Dec 2020 12:04:01 +0100 Subject: [PATCH 58/92] Move transaction_with_delayed_retry/3 into mongoose_rdbms --- src/mam/mod_mam_rdbms_prefs.erl | 27 ++++----------------------- src/rdbms/mongoose_rdbms.erl | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/mam/mod_mam_rdbms_prefs.erl b/src/mam/mod_mam_rdbms_prefs.erl index bb69116ec9..2173b94796 100644 --- a/src/mam/mod_mam_rdbms_prefs.erl +++ b/src/mam/mod_mam_rdbms_prefs.erl @@ -192,12 +192,14 @@ set_prefs(_Result, Host, UserID, _ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> set_prefs1(Host, UserID, DefaultMode, AlwaysJIDs, NeverJIDs) -> Rows = prefs_to_rows(UserID, DefaultMode, AlwaysJIDs, NeverJIDs), - run_transaction_or_retry_on_abort(Host, fun() -> + %% MySQL sometimes aborts transaction with reason: + %% "Deadlock found when trying to get lock; try restarting transaction" + mongoose_rdbms:transaction_with_delayed_retry(Host, fun() -> {updated, _} = mongoose_rdbms:execute(Host, mam_prefs_delete, [UserID]), [mongoose_rdbms:execute(Host, mam_prefs_insert, Row) || Row <- Rows], ok - end, UserID, 5), + end, #{user_id => UserID, retries => 5, delay => 100}), ok. -spec get_prefs(mod_mam:preference(), _Host :: jid:server(), @@ -232,27 +234,6 @@ query_behaviour(Host, UserID, BRemLJID, BRemLBareJID) -> %% ---------------------------------------------------------------------- %% Helpers -%% Possible error with mysql -%% Reason "Deadlock found when trying to get lock; try restarting transaction" -%% triggered mod_mam_utils:error_on_sql_error -run_transaction_or_retry_on_abort(Host, F, UserID, Retries) -> - Result = mongoose_rdbms:sql_transaction(Host, F), - case Result of - {atomic, _} -> - Result; - {aborted, Reason} when Retries > 0 -> - ?LOG_WARNING(#{what => mam_transaction_aborted, - text => <<"Transaction aborted. Restart">>, - user_id => UserID, reason => Reason, retries => Retries}), - timer:sleep(100), - run_transaction_or_retry_on_abort(Host, F, UserID, Retries-1); - _ -> - ?LOG_ERROR(#{what => mam_transaction_failed, - text => <<"Transaction failed. Do not restart">>, - user_id => UserID, reason => Result, retries => Retries}), - erlang:error({transaction_failed, #{user_id => UserID, result => Result}}) - end. - -spec encode_behaviour(always | never | roster) -> binary(). encode_behaviour(roster) -> <<"R">>; encode_behaviour(always) -> <<"A">>; diff --git a/src/rdbms/mongoose_rdbms.erl b/src/rdbms/mongoose_rdbms.erl index d452997a2f..b3a33ede95 100644 --- a/src/rdbms/mongoose_rdbms.erl +++ b/src/rdbms/mongoose_rdbms.erl @@ -82,6 +82,7 @@ sql_query/2, sql_query_t/1, sql_transaction/2, + transaction_with_delayed_retry/3, sql_dirty/2, to_bool/1, db_engine/1, @@ -238,6 +239,32 @@ sql_transaction(Host, Queries) when is_list(Queries) -> sql_transaction(Host, F) when is_function(F) -> sql_call(Host, {sql_transaction, F}). +%% This function allows to specify delay between retries. +-spec transaction_with_delayed_retry(server(), fun() | maybe_improper_list(), map()) -> transaction_result(). +transaction_with_delayed_retry(Host, F, Info) -> + Retries = maps:get(retries, Info), + Delay = maps:get(delay, Info), + do_transaction_with_delayed_retry(Host, F, Retries, Delay, Info). + +do_transaction_with_delayed_retry(Host, F, Retries, Delay, Info) -> + Result = mongoose_rdbms:sql_transaction(Host, F), + case Result of + {atomic, _} -> + Result; + {aborted, Reason} when Retries > 0 -> + ?LOG_WARNING(Info#{what => rdbms_transaction_aborted, + text => <<"Transaction aborted. Restart">>, + reason => Reason, retries_left => Retries}), + timer:sleep(Delay), + do_transaction_with_delayed_retry(Host, F, Retries - 1, Delay, Info); + _ -> + Err = Info#{what => mam_transaction_failed, + text => <<"Transaction failed. Do not restart">>, + reason => Result}, + ?LOG_ERROR(Err), + erlang:error(Err) + end. + -spec sql_dirty(server(), fun()) -> any() | no_return(). sql_dirty(Host, F) when is_function(F) -> case sql_call(Host, {sql_dirty, F}) of From 1eae909eef22cc56ff0f8ec084f3bb709475e557 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 18 Dec 2020 13:17:13 +0100 Subject: [PATCH 59/92] Move sql queries into mod_roster_rdbms --- src/mod_roster_rdbms.erl | 145 +++++++++++++++++++++++++---- src/rdbms/rdbms_queries.erl | 181 ------------------------------------ 2 files changed, 127 insertions(+), 199 deletions(-) diff --git a/src/mod_roster_rdbms.erl b/src/mod_roster_rdbms.erl index 5df5a220c6..4f29edc410 100644 --- a/src/mod_roster_rdbms.erl +++ b/src/mod_roster_rdbms.erl @@ -48,7 +48,7 @@ transaction(LServer, F) -> -> binary() | error. read_roster_version(LUser, LServer) -> Username = mongoose_rdbms:escape_string(LUser), - case rdbms_queries:get_roster_version(LServer, Username) + case get_roster_version(LServer, Username) of {selected, [{Version}]} -> Version; {selected, []} -> error @@ -59,20 +59,19 @@ write_roster_version(LUser, LServer, InTransaction, Ver) -> EVer = mongoose_rdbms:escape_string(Ver), case InTransaction of true -> - rdbms_queries:set_roster_version(Username, EVer); + set_roster_version(Username, EVer); _ -> rdbms_queries:sql_transaction(LServer, fun () -> - rdbms_queries:set_roster_version(Username, - EVer) + set_roster_version(Username, EVer) end) end. get_roster(LUser, LServer) -> Username = mongoose_rdbms:escape_string(LUser), - try rdbms_queries:get_roster(LServer, Username) of + try get_roster_sql(LServer, Username) of {selected, Items} when is_list(Items) -> - {selected, JIDGroups} = rdbms_queries:get_roster_jid_groups(LServer, Username), + {selected, JIDGroups} = get_roster_jid_groups(LServer, Username), GroupsDict = lists:foldl(fun ({J, G}, Acc) -> dict:append(J, G, Acc) end, @@ -113,9 +112,9 @@ do_get_roster_entry(LUser, LServer, LJID, FuncName) -> SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), {selected, Res} = case FuncName of get_roster_by_jid -> - rdbms_queries:get_roster_by_jid(LServer, Username, SJID); + get_roster_by_jid(LServer, Username, SJID); get_roster_by_jid_t -> - rdbms_queries:get_roster_by_jid_t(LServer, Username, SJID) + get_roster_by_jid_t(LServer, Username, SJID) end, case Res of [] -> @@ -158,7 +157,7 @@ get_roster_entry_t(LUser, LServer, LJID, full) -> get_subscription_lists(_, LUser, LServer) -> Username = mongoose_rdbms:escape_string(LUser), - try rdbms_queries:get_roster(LServer, Username) of + try get_roster_sql(LServer, Username) of {selected, Items} when is_list(Items) -> Items; Other -> @@ -176,12 +175,11 @@ roster_subscribe_t(LUser, LServer, LJID, Item) -> ItemVals = record_to_string(Item), Username = mongoose_rdbms:escape_string(LUser), SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - rdbms_queries:roster_subscribe(LServer, Username, SJID, - ItemVals). + roster_subscribe(LServer, Username, SJID, ItemVals). remove_user(LUser, LServer) -> Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:del_user_roster_t(LServer, Username), + del_user_roster_t(LServer, Username), ok. update_roster_t(LUser, LServer, LJID, Item) -> @@ -189,12 +187,12 @@ update_roster_t(LUser, LServer, LJID, Item) -> SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), ItemVals = record_to_string(Item), ItemGroups = groups_to_string(Item), - rdbms_queries:update_roster(LServer, Username, SJID, ItemVals, ItemGroups). + update_roster(LServer, Username, SJID, ItemVals, ItemGroups). del_roster_t(LUser, LServer, LJID) -> Username = mongoose_rdbms:escape_string(LUser), SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - rdbms_queries:del_roster(LServer, Username, SJID). + del_roster(LServer, Username, SJID). raw_to_record(LServer, {User, SJID, Nick, SSubscription, SAsk, SAskMessage, @@ -237,9 +235,9 @@ read_subscription_and_groups(LUser, LServer, LJID, GSFunc, GRFunc) -> SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), SubResult = case GSFunc of get_subscription -> - catch rdbms_queries:get_subscription(LServer, Username, SJID); + catch get_subscription(LServer, Username, SJID); get_subscription_t -> - catch rdbms_queries:get_subscription_t(LServer, Username, SJID) + catch get_subscription_t(LServer, Username, SJID) end, case SubResult of {selected, [{SSubscription}]} -> @@ -251,9 +249,9 @@ read_subscription_and_groups(LUser, LServer, LJID, GSFunc, GRFunc) -> end, GRResult = case GRFunc of get_rostergroup_by_jid -> - catch rdbms_queries:get_rostergroup_by_jid(LServer, Username, SJID); + catch get_rostergroup_by_jid(LServer, Username, SJID); get_rostergroup_by_jid_t -> - catch rdbms_queries:get_rostergroup_by_jid_t(LServer, Username, SJID) + catch get_rostergroup_by_jid_t(LServer, Username, SJID) end, Groups = case GRResult of {selected, JGrps} when is_list(JGrps) -> @@ -312,3 +310,114 @@ groups_to_string(#roster{us = {User, _Server}, [[Username, SJID, G] | Acc] end, [], Groups). + + +get_roster_version(LServer, LUser) -> + mongoose_rdbms:sql_query( + LServer, + [<<"select version from roster_version " + "where username=">>, mongoose_rdbms:use_escaped_string(LUser)]). + +set_roster_version(LUser, Version) -> + mongoose_rdbms:update_t( + <<"roster_version">>, + [<<"username">>, <<"version">>], + [LUser, Version], + [<<"username = ">>, mongoose_rdbms:use_escaped_string(LUser)]). + +get_roster_sql(LServer, Username) -> + mongoose_rdbms:sql_query( + LServer, + [<<"select username, jid, nick, subscription, ask, " + "askmessage, server, subscribe, type from rosterusers " + "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). + +get_roster_jid_groups(LServer, Username) -> + mongoose_rdbms:sql_query( + LServer, + [<<"select jid, grp from rostergroups " + "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). + +del_user_roster_t(LServer, Username) -> + mongoose_rdbms:sql_transaction( + LServer, + fun() -> + mongoose_rdbms:sql_query_t( + [<<"delete from rosterusers " + "where username=">>, mongoose_rdbms:use_escaped_string(Username)]), + mongoose_rdbms:sql_query_t( + [<<"delete from rostergroups " + "where username=">>, mongoose_rdbms:use_escaped_string(Username)]) + end). + + +q_get_roster(Username, SJID) -> + [<<"select username, jid, nick, subscription, " + "ask, askmessage, server, subscribe, type from rosterusers " + "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " + "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. + +get_roster_by_jid(LServer, Username, SJID) -> + mongoose_rdbms:sql_query(LServer, q_get_roster(Username, SJID)). + +get_roster_by_jid_t(_LServer, Username, SJID) -> + mongoose_rdbms:sql_query_t(q_get_roster(Username, SJID)). + +q_get_rostergroup(Username, SJID) -> + [<<"select grp from rostergroups " + "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " + "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. + +get_rostergroup_by_jid(LServer, Username, SJID) -> + mongoose_rdbms:sql_query(LServer, q_get_rostergroup(Username, SJID)). + +get_rostergroup_by_jid_t(_LServer, Username, SJID) -> + mongoose_rdbms:sql_query_t(q_get_rostergroup(Username, SJID)). + +del_roster(_LServer, Username, SJID) -> + mongoose_rdbms:sql_query_t( + [<<"delete from rosterusers " + "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " + "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), + mongoose_rdbms:sql_query_t( + [<<"delete from rostergroups " + "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " + "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]). + +update_roster(_LServer, Username, SJID, ItemVals, ItemGroups) -> + mongoose_rdbms:update_t(<<"rosterusers">>, + [<<"username">>, <<"jid">>, <<"nick">>, <<"subscription">>, <<"ask">>, + <<"askmessage">>, <<"server">>, <<"subscribe">>, <<"type">>], + ItemVals, + [<<"username=">>, mongoose_rdbms:use_escaped_string(Username), + <<" and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), + mongoose_rdbms:sql_query_t( + [<<"delete from rostergroups " + "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " + "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), + lists:foreach(fun(ItemGroup) -> + mongoose_rdbms:sql_query_t( + [<<"insert into rostergroups(username, jid, grp) " + "values (">>, mongoose_rdbms:join_escaped(ItemGroup), ");"]) + end, + ItemGroups). + +roster_subscribe(_LServer, Username, SJID, ItemVals) -> + mongoose_rdbms:update_t(<<"rosterusers">>, + [<<"username">>, <<"jid">>, <<"nick">>, <<"subscription">>, <<"ask">>, + <<"askmessage">>, <<"server">>, <<"subscribe">>, <<"type">>], + ItemVals, + [<<"username=">>, mongoose_rdbms:use_escaped_string(Username), + <<" and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]). + +q_get_subscription(Username, SJID) -> + [<<"select subscription from rosterusers " + "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " + "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. + +get_subscription(LServer, Username, SJID) -> + mongoose_rdbms:sql_query( LServer, q_get_subscription(Username, SJID)). + +get_subscription_t(_LServer, Username, SJID) -> + mongoose_rdbms:sql_query_t(q_get_subscription(Username, SJID)). + diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index 2db3da8f9d..7603586bb2 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -50,24 +50,6 @@ users_number/2, get_users_without_scram/2, get_users_without_scram_count/1, - get_average_roster_size/1, - get_average_rostergroup_size/1, - clear_rosters/1, - get_roster/2, - get_roster_jid_groups/2, - get_roster_groups/3, - del_user_roster_t/2, - get_roster_by_jid/3, - get_roster_by_jid_t/3, - get_rostergroup_by_jid/3, - get_rostergroup_by_jid_t/3, - del_roster/3, - del_roster_sql/2, - update_roster/5, - update_roster_sql/4, - roster_subscribe/4, - get_subscription/3, - get_subscription_t/3, get_default_privacy_list/2, get_default_privacy_list_t/1, count_privacy_lists/1, @@ -85,8 +67,6 @@ set_privacy_list/2, del_privacy_lists/3, count_records_where/3, - get_roster_version/2, - set_roster_version/2, prepare_offline_message/7, push_offline_messages/2, pop_offline_messages/4, @@ -437,153 +417,6 @@ get_users_without_scram_count(LServer) -> LServer, [<<"select count(*) from users where pass_details is null">>]). -get_average_roster_size(Server) -> - mongoose_rdbms:sql_query( - Server, - [<<"select avg(items) from " - "(select count(*) as items from rosterusers group by username) as items;">>]). - -get_average_rostergroup_size(Server) -> - mongoose_rdbms:sql_query( - Server, - [<<"select avg(roster) from " - "(select count(*) as roster from rostergroups group by username) as roster;">>]). - -clear_rosters(Server) -> - mongoose_rdbms:sql_transaction( - Server, - fun() -> - mongoose_rdbms:sql_query_t( - [<<"delete from rosterusers;">>]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups;">>]) - end). - -get_roster(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select username, jid, nick, subscription, ask, " - "askmessage, server, subscribe, type from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_roster_jid_groups(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select jid, grp from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_roster_groups(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t( - [<<"select grp from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID), ";"]). - -del_user_roster_t(LServer, Username) -> - mongoose_rdbms:sql_transaction( - LServer, - fun() -> - mongoose_rdbms:sql_query_t( - [<<"delete from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]) - end). - -q_get_roster(Username, SJID) -> - [<<"select username, jid, nick, subscription, " - "ask, askmessage, server, subscribe, type from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. - -get_roster_by_jid(LServer, Username, SJID) -> - mongoose_rdbms:sql_query(LServer, q_get_roster(Username, SJID)). - -get_roster_by_jid_t(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t(q_get_roster(Username, SJID)). - -q_get_rostergroup(Username, SJID) -> - [<<"select grp from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. - -get_rostergroup_by_jid(LServer, Username, SJID) -> - mongoose_rdbms:sql_query(LServer, q_get_rostergroup(Username, SJID)). - -get_rostergroup_by_jid_t(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t(q_get_rostergroup(Username, SJID)). - -del_roster(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t( - [<<"delete from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]). - -del_roster_sql(Username, SJID) -> - [[<<"delete from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)], - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]]. - -update_roster(_LServer, Username, SJID, ItemVals, ItemGroups) -> - update_t(<<"rosterusers">>, - [<<"username">>, <<"jid">>, <<"nick">>, <<"subscription">>, <<"ask">>, - <<"askmessage">>, <<"server">>, <<"subscribe">>, <<"type">>], - ItemVals, - [<<"username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), - lists:foreach(fun(ItemGroup) -> - mongoose_rdbms:sql_query_t( - [<<"insert into rostergroups(username, jid, grp) " - "values (">>, join_escaped(ItemGroup), ");"]) - end, - ItemGroups). - -update_roster_sql(Username, SJID, ItemVals, ItemGroups) -> - [[<<"delete from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)], - [<<"insert into rosterusers(" - "username, jid, nick, " - "subscription, ask, askmessage, " - "server, subscribe, type) " - " values (">>, join_escaped(ItemVals), ");"], - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID), ";"]] ++ - [[<<"insert into rostergroups(username, jid, grp) " - "values (">>, join_escaped(ItemGroup), ");"] || - ItemGroup <- ItemGroups]. - -roster_subscribe(_LServer, Username, SJID, ItemVals) -> - update_t(<<"rosterusers">>, - [<<"username">>, <<"jid">>, <<"nick">>, <<"subscription">>, <<"ask">>, - <<"askmessage">>, <<"server">>, <<"subscribe">>, <<"type">>], - ItemVals, - [<<"username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]). - -q_get_subscription(Username, SJID) -> - [<<"select subscription from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. - -get_subscription(LServer, Username, SJID) -> - mongoose_rdbms:sql_query( LServer, q_get_subscription(Username, SJID)). - -get_subscription_t(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t(q_get_subscription(Username, SJID)). - get_default_privacy_list(LServer, Username) -> mongoose_rdbms:sql_query( LServer, @@ -706,20 +539,6 @@ count_records_where(LServer, Table, WhereClause) -> [<<"select count(*) from ">>, Table, " ", WhereClause, ";"]). -get_roster_version(LServer, LUser) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select version from roster_version " - "where username=">>, mongoose_rdbms:use_escaped_string(LUser)]). - -set_roster_version(LUser, Version) -> - update_t( - <<"roster_version">>, - [<<"username">>, <<"version">>], - [LUser, Version], - [<<"username = ">>, mongoose_rdbms:use_escaped_string(LUser)]). - - pop_offline_messages(LServer, SUser, SServer, STimeStamp) -> SelectSQL = select_offline_messages_sql(SUser, SServer, STimeStamp), DeleteSQL = delete_offline_messages_sql(SUser, SServer), From 30db492b368ccf8d85bd601c3b8e9692776f17f0 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 18 Dec 2020 13:23:49 +0100 Subject: [PATCH 60/92] Remove mod_mam_utils:success_sql_query/2 --- src/mam/mod_mam_utils.erl | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/mam/mod_mam_utils.erl b/src/mam/mod_mam_utils.erl index aaa1f0deb0..0053aa9656 100644 --- a/src/mam/mod_mam_utils.erl +++ b/src/mam/mod_mam_utils.erl @@ -68,9 +68,6 @@ -export([jid_to_opt_binary/2, expand_minified_jid/2]). -%% SQL --export([success_sql_query/2]). - %% Other -export([maybe_integer/2, maybe_min/2, @@ -1093,19 +1090,6 @@ is_jid_in_user_roster(#jid{lserver = LServer} = ToJID, {Subscription, _G} = mongoose_hooks:roster_get_jid_info(LServer, {none, []}, ToJID, RemBareJID), Subscription == from orelse Subscription == both. - --spec success_sql_query(atom() | jid:server(), mongoose_rdbms:sql_query()) -> any(). -success_sql_query(HostOrConn, Query) -> - Result = mongoose_rdbms:sql_query(HostOrConn, Query), - error_on_sql_error(HostOrConn, Query, Result). - -error_on_sql_error(HostOrConn, Query, {error, Reason}) -> - ?LOG_ERROR(#{what => mam_sql_error, - host => HostOrConn, query => Query, reason => Reason}), - error({sql_error, Reason}); -error_on_sql_error(_HostOrConn, _Query, Result) -> - Result. - %% @doc Returns a UUIDv4 canonical form binary. -spec wrapper_id() -> binary(). wrapper_id() -> From 06bb90c6f9723499a8b151deac2ce8370fc9d5c7 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 18 Dec 2020 15:37:10 +0100 Subject: [PATCH 61/92] Use prepared queries in mod_roster Merge logic for queries with and without transactions. There is no need to do transaction checking in the roster code. This is already done in mongoose_rdbms. --- src/mod_roster_rdbms.erl | 519 ++++++++++++++++----------------------- 1 file changed, 209 insertions(+), 310 deletions(-) diff --git a/src/mod_roster_rdbms.erl b/src/mod_roster_rdbms.erl index 4f29edc410..34fee4522b 100644 --- a/src/mod_roster_rdbms.erl +++ b/src/mod_roster_rdbms.erl @@ -36,51 +36,100 @@ -export([raw_to_record/2]). -spec init(jid:server(), list()) -> ok. -init(_Host, _Opts) -> +init(Host, _Opts) -> + prepare_queries(Host), ok. +prepare_queries(Host) -> + mongoose_rdbms:prepare(roster_group_insert, rostergroups, [username, jid, grp], + <<"INSERT INTO rostergroups(username, jid, grp) " + "VALUES (?, ?, ?)">>), + mongoose_rdbms:prepare(roster_version_get, roster_version, [username], + <<"SELECT version FROM roster_version " + "WHERE username=?">>), + mongoose_rdbms:prepare(roster_get, rosterusers, [username], + <<"SELECT ", (roster_fields())/binary, + " FROM rosterusers WHERE username=?">>), + mongoose_rdbms:prepare(roster_get_by_jid, rostergroups, [username, jid], + <<"SELECT ", (roster_fields())/binary, + " FROM rosterusers WHERE username=? AND jid=?">>), + mongoose_rdbms:prepare(roster_group_get, rostergroups, [username], + <<"SELECT jid, grp FROM rostergroups WHERE username=?">>), + mongoose_rdbms:prepare(roster_sub_get_by_jid, rosterusers, [username, jid], + <<"SELECT subscription FROM rosterusers " + "WHERE username=? AND jid=?">>), + mongoose_rdbms:prepare(roster_group_get_by_jid, rostergroups, [username, jid], + <<"SELECT grp FROM rostergroups " + "WHERE username=? AND jid=?">>), + mongoose_rdbms:prepare(roster_delete, rosterusers, [username], + <<"DELETE FROM rosterusers WHERE username=?">>), + mongoose_rdbms:prepare(roster_group_delete, rostergroups, [username], + <<"DELETE FROM rostergroups WHERE username=?">>), + mongoose_rdbms:prepare(roster_delete_by_jid, rosterusers, [username, jid], + <<"DELETE FROM rosterusers WHERE username=? AND jid=?">>), + mongoose_rdbms:prepare(roster_group_delete_by_jid, rostergroups, [username, jid], + <<"DELETE FROM rostergroups WHERE username=? AND jid=?">>), + prepare_roster_upsert(Host), + prepare_version_upsert(Host), + ok. + +roster_fields() -> + <<"username, jid, nick, subscription, ask, " + "askmessage, server, subscribe, type">>. + +prepare_roster_upsert(Host) -> + Fields = [<<"nick">>, <<"subscription">>, <<"ask">>, + <<"askmessage">>, <<"server">>, <<"subscribe">>, <<"type">>], + Filter = [<<"username">>, <<"jid">>], + rdbms_queries:prepare_upsert(Host, roster_upsert, rosterusers, + Filter ++ Fields, Fields, Filter). + +prepare_version_upsert(Host) -> + Fields = [<<"version">>], + Filter = [<<"username">>], + rdbms_queries:prepare_upsert(Host, roster_version_upsert, roster_version, + Filter ++ Fields, Fields, Filter). + +%% Query Helpers + +execute_roster_get(LServer, LUser) -> + mongoose_rdbms:execute(LServer, roster_get, [LUser]). + +roster_upsert(Host, LUser, BinJID, RosterRow) -> + [_LUser, _BinJID|Rest] = RosterRow, + InsertParams = RosterRow, + UpdateParams = Rest, + UniqueKeyValues = [LUser, BinJID], + rdbms_queries:execute_upsert(Host, roster_upsert, InsertParams, UpdateParams, UniqueKeyValues). + +version_upsert(Host, LUser, Version) -> + InsertParams = [LUser, Version], + UpdateParams = [Version], + UniqueKeyValues = [LUser], + rdbms_queries:execute_upsert(Host, roster_version_upsert, InsertParams, UpdateParams, UniqueKeyValues). + +%% API functions + -spec transaction(LServer :: jid:lserver(), F :: fun()) -> {aborted, Reason :: any()} | {atomic, Result :: any()}. transaction(LServer, F) -> mongoose_rdbms:sql_transaction(LServer, F). --spec read_roster_version(jid:luser(), jid:lserver()) --> binary() | error. +-spec read_roster_version(jid:luser(), jid:lserver()) -> binary() | error. read_roster_version(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - case get_roster_version(LServer, Username) - of + case mongoose_rdbms:execute(LServer, roster_version_get, [LUser]) of {selected, [{Version}]} -> Version; {selected, []} -> error end. -write_roster_version(LUser, LServer, InTransaction, Ver) -> - Username = mongoose_rdbms:escape_string(LUser), - EVer = mongoose_rdbms:escape_string(Ver), - case InTransaction of - true -> - set_roster_version(Username, EVer); - _ -> - rdbms_queries:sql_transaction(LServer, - fun () -> - set_roster_version(Username, EVer) - end) - end. +write_roster_version(LUser, LServer, _InTransaction, Ver) -> + version_upsert(LServer, LUser, Ver). get_roster(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - try get_roster_sql(LServer, Username) of - {selected, Items} when is_list(Items) -> - {selected, JIDGroups} = get_roster_jid_groups(LServer, Username), - GroupsDict = lists:foldl(fun ({J, G}, Acc) -> - dict:append(J, G, Acc) - end, - dict:new(), JIDGroups), - RItems = lists:flatmap(fun (I) -> - raw_to_record_with_group(LServer, I, GroupsDict) - end, - Items), - RItems; + try execute_roster_get(LServer, LUser) of + {selected, Rows} -> + {selected, GroupRows} = mongoose_rdbms:execute(LServer, roster_group_get, [LUser]), + decode_roster_rows(LServer, Rows, GroupRows); _ -> [] catch Class:Reason:StackTrace -> ?LOG_ERROR(#{what => get_roster_failed, class => Class, reason => Reason, @@ -88,48 +137,26 @@ get_roster(LUser, LServer) -> [] end. -raw_to_record_with_group(LServer, I, GroupsDict) -> - case raw_to_record(LServer, I) of - %% Bad JID in database: - error -> []; - R -> - SJID = jid:to_binary(R#roster.jid), - Groups = case dict:find(SJID, GroupsDict) of - {ok, Gs} -> Gs; - error -> [] - end, - [R#roster{groups = Groups}] - end. +get_roster_entry_t(LUser, LServer, LJID) -> + get_roster_entry(LUser, LServer, LJID). get_roster_entry(LUser, LServer, LJID) -> - do_get_roster_entry(LUser, LServer, LJID, get_roster_by_jid). - -get_roster_entry_t(LUser, LServer, LJID) -> - do_get_roster_entry(LUser, LServer, LJID, get_roster_by_jid_t). - -do_get_roster_entry(LUser, LServer, LJID, FuncName) -> - Username = mongoose_rdbms:escape_string(LUser), - SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - {selected, Res} = case FuncName of - get_roster_by_jid -> - get_roster_by_jid(LServer, Username, SJID); - get_roster_by_jid_t -> - get_roster_by_jid_t(LServer, Username, SJID) - end, - case Res of - [] -> - does_not_exist; - [I] -> - R = raw_to_record(LServer, I), - case R of - %% Bad JID in database: - error -> - #roster{usj = {LUser, LServer, LJID}, - us = {LUser, LServer}, jid = LJID}; - _ -> - R#roster{usj = {LUser, LServer, LJID}, - us = {LUser, LServer}, jid = LJID} - end + BinJID = jid:to_binary(LJID), + {selected, Rows} = mongoose_rdbms:execute(LServer, roster_get_by_jid, [LUser, BinJID]), + decode_roster_entry_rows(LUser, LServer, LJID, Rows). + +decode_roster_entry_rows(_LUser, _LServer, _LJID, []) -> + does_not_exist; +decode_roster_entry_rows(LUser, LServer, LJID, [Row]) -> + Rec = raw_to_record(LServer, Row), + USJ = {LUser, LServer, LJID}, + US = {LUser, LServer}, + case Rec of + %% Bad JID in database: + error -> + #roster{usj = USJ, us = US, jid = LJID}; + _ -> + Rec#roster{usj = USJ, us = US, jid = LJID} end. get_roster_entry(LUser, LServer, LJID, full) -> @@ -144,22 +171,12 @@ get_roster_entry(LUser, LServer, LJID, full) -> end. get_roster_entry_t(LUser, LServer, LJID, full) -> - case get_roster_entry_t(LUser, LServer, LJID) of - does_not_exist -> does_not_exist; - Rentry -> - case read_subscription_and_groups_t(LUser, LServer, LJID) of - error -> error; - {Subscription, Groups} -> - Rentry#roster{subscription = Subscription, groups = Groups} - end - end. - + get_roster_entry(LUser, LServer, LJID, full). get_subscription_lists(_, LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - try get_roster_sql(LServer, Username) of - {selected, Items} when is_list(Items) -> - Items; + try execute_roster_get(LServer, LUser) of + {selected, Rows} -> + Rows; Other -> ?LOG_ERROR(#{what => get_subscription_lists_failed, reason => Other, user => LUser, host => LServer}), @@ -172,97 +189,50 @@ get_subscription_lists(_, LUser, LServer) -> end. roster_subscribe_t(LUser, LServer, LJID, Item) -> - ItemVals = record_to_string(Item), - Username = mongoose_rdbms:escape_string(LUser), - SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - roster_subscribe(LServer, Username, SJID, ItemVals). + BinJID = jid:to_binary(LJID), + RosterRow = record_to_row(Item), + roster_upsert(LServer, LUser, BinJID, RosterRow). remove_user(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - del_user_roster_t(LServer, Username), + F = fun() -> + mongoose_rdbms:execute(LServer, roster_delete, [LUser]), + mongoose_rdbms:execute(LServer, roster_group_delete, [LUser]) + end, + mongoose_rdbms:sql_transaction(LServer, F), ok. update_roster_t(LUser, LServer, LJID, Item) -> - Username = mongoose_rdbms:escape_string(LUser), - SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - ItemVals = record_to_string(Item), - ItemGroups = groups_to_string(Item), - update_roster(LServer, Username, SJID, ItemVals, ItemGroups). + BinJID = jid:to_binary(LJID), + RosterRow = record_to_row(Item), + GroupRows = groups_to_rows(Item), + roster_upsert(LServer, LUser, BinJID, RosterRow), + mongoose_rdbms:execute(LServer, roster_group_delete_by_jid, [LUser, BinJID]), + [mongoose_rdbms:execute(LServer, roster_group_insert, GroupRow) + || GroupRow <- GroupRows], + ok. del_roster_t(LUser, LServer, LJID) -> - Username = mongoose_rdbms:escape_string(LUser), - SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - del_roster(LServer, Username, SJID). - -raw_to_record(LServer, - {User, SJID, Nick, SSubscription, SAsk, SAskMessage, - _SServer, _SSubscribe, _SType}) -> - case jid:from_binary(SJID) of - error -> error; - JID -> - LJID = jid:to_lower(JID), - Subscription = case SSubscription of - <<"B">> -> both; - <<"T">> -> to; - <<"F">> -> from; - _ -> none - end, - Ask = case SAsk of - <<"S">> -> subscribe; - <<"U">> -> unsubscribe; - <<"B">> -> both; - <<"O">> -> out; - <<"I">> -> in; - _ -> none - end, - #roster{usj = {User, LServer, LJID}, - us = {User, LServer}, jid = LJID, name = Nick, - subscription = Subscription, ask = Ask, - askmessage = SAskMessage} - end. - + BinJID = jid:to_binary(LJID), + mongoose_rdbms:execute(LServer, roster_delete_by_jid, [LUser, BinJID]), + mongoose_rdbms:execute(LServer, roster_group_delete_by_jid, [LUser, BinJID]). read_subscription_and_groups(LUser, LServer, LJID) -> - read_subscription_and_groups(LUser, LServer, LJID, get_subscription, - get_rostergroup_by_jid). - -read_subscription_and_groups_t(LUser, LServer, LJID) -> - read_subscription_and_groups(LUser, LServer, LJID, get_subscription_t, - get_rostergroup_by_jid_t). - -read_subscription_and_groups(LUser, LServer, LJID, GSFunc, GRFunc) -> - Username = mongoose_rdbms:escape_string(LUser), - SJID = mongoose_rdbms:escape_string(jid:to_binary(LJID)), - SubResult = case GSFunc of - get_subscription -> - catch get_subscription(LServer, Username, SJID); - get_subscription_t -> - catch get_subscription_t(LServer, Username, SJID) - end, + BinJID = jid:to_binary(LJID), + SubResult = mongoose_rdbms:execute(LServer, roster_sub_get_by_jid, [LUser, BinJID]), case SubResult of - {selected, [{SSubscription}]} -> - Subscription = case SSubscription of - <<"B">> -> both; - <<"T">> -> to; - <<"F">> -> from; - _ -> none - end, - GRResult = case GRFunc of - get_rostergroup_by_jid -> - catch get_rostergroup_by_jid(LServer, Username, SJID); - get_rostergroup_by_jid_t -> - catch get_rostergroup_by_jid_t(LServer, Username, SJID) - end, - Groups = case GRResult of - {selected, JGrps} when is_list(JGrps) -> - [JGrp || {JGrp} <- JGrps]; - _ -> - ?LOG_ERROR(#{what => read_subscription_and_groups_failed, - reason => GRResult, user => LUser, - host => LServer}), - [] - end, - {Subscription, Groups}; + {selected, [{ExtSubscription}]} -> + GroupResult = mongoose_rdbms:execute(LServer, roster_group_get_by_jid, [LUser, BinJID]), + Subscription = decode_subscription(ExtSubscription), + case GroupResult of + {selected, GroupRows} -> + Groups = [JGrp || {JGrp} <- GroupRows], + {Subscription, Groups}; + _ -> + ?LOG_ERROR(#{what => read_subscription_and_groups_failed, + reason => GroupResult, user => LUser, + host => LServer}), + error + end; E -> ?LOG_ERROR(#{what => read_subscription_and_groups_failed, reason => E, user => LUser, host => LServer}), @@ -273,151 +243,80 @@ read_subscription_and_groups(LUser, LServer, LJID, GSFunc, GRFunc) -> %% Helper functions %%============================================================================== -record_to_string(#roster{us = {User, _Server}, - jid = JID, name = Name, subscription = Subscription, - ask = Ask, askmessage = AskMessage}) -> - Username = mongoose_rdbms:escape_string(User), - SJID = - mongoose_rdbms:escape_string(jid:to_binary(jid:to_lower(JID))), - Nick = mongoose_rdbms:escape_string(Name), - SSubscription = mongoose_rdbms:escape_string(case Subscription of - both -> <<"B">>; - to -> <<"T">>; - from -> <<"F">>; - none -> <<"N">> - end), - SAsk = mongoose_rdbms:escape_string(case Ask of - subscribe -> <<"S">>; - unsubscribe -> <<"U">>; - both -> <<"B">>; - out -> <<"O">>; - in -> <<"I">>; - none -> <<"N">> - end), - SAskMessage = mongoose_rdbms:escape_string(AskMessage), - [Username, SJID, Nick, SSubscription, SAsk, SAskMessage, - mongoose_rdbms:escape_string(<<"N">>), - mongoose_rdbms:escape_string(<<"">>), - mongoose_rdbms:escape_string(<<"item">>)]. - -groups_to_string(#roster{us = {User, _Server}, - jid = JID, groups = Groups}) -> - Username = mongoose_rdbms:escape_string(User), - SJID = mongoose_rdbms:escape_string(jid:to_binary(jid:to_lower(JID))), - lists:foldl(fun (<<"">>, Acc) -> Acc; - (Group, Acc) -> - G = mongoose_rdbms:escape_string(Group), - [[Username, SJID, G] | Acc] - end, - [], Groups). - - -get_roster_version(LServer, LUser) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select version from roster_version " - "where username=">>, mongoose_rdbms:use_escaped_string(LUser)]). - -set_roster_version(LUser, Version) -> - mongoose_rdbms:update_t( - <<"roster_version">>, - [<<"username">>, <<"version">>], - [LUser, Version], - [<<"username = ">>, mongoose_rdbms:use_escaped_string(LUser)]). - -get_roster_sql(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select username, jid, nick, subscription, ask, " - "askmessage, server, subscribe, type from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_roster_jid_groups(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select jid, grp from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -del_user_roster_t(LServer, Username) -> - mongoose_rdbms:sql_transaction( - LServer, - fun() -> - mongoose_rdbms:sql_query_t( - [<<"delete from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]) - end). - - -q_get_roster(Username, SJID) -> - [<<"select username, jid, nick, subscription, " - "ask, askmessage, server, subscribe, type from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. - -get_roster_by_jid(LServer, Username, SJID) -> - mongoose_rdbms:sql_query(LServer, q_get_roster(Username, SJID)). - -get_roster_by_jid_t(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t(q_get_roster(Username, SJID)). - -q_get_rostergroup(Username, SJID) -> - [<<"select grp from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. - -get_rostergroup_by_jid(LServer, Username, SJID) -> - mongoose_rdbms:sql_query(LServer, q_get_rostergroup(Username, SJID)). - -get_rostergroup_by_jid_t(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t(q_get_rostergroup(Username, SJID)). - -del_roster(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t( - [<<"delete from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]). - -update_roster(_LServer, Username, SJID, ItemVals, ItemGroups) -> - mongoose_rdbms:update_t(<<"rosterusers">>, - [<<"username">>, <<"jid">>, <<"nick">>, <<"subscription">>, <<"ask">>, - <<"askmessage">>, <<"server">>, <<"subscribe">>, <<"type">>], - ItemVals, - [<<"username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), - mongoose_rdbms:sql_query_t( - [<<"delete from rostergroups " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]), - lists:foreach(fun(ItemGroup) -> - mongoose_rdbms:sql_query_t( - [<<"insert into rostergroups(username, jid, grp) " - "values (">>, mongoose_rdbms:join_escaped(ItemGroup), ");"]) - end, - ItemGroups). - -roster_subscribe(_LServer, Username, SJID, ItemVals) -> - mongoose_rdbms:update_t(<<"rosterusers">>, - [<<"username">>, <<"jid">>, <<"nick">>, <<"subscription">>, <<"ask">>, - <<"askmessage">>, <<"server">>, <<"subscribe">>, <<"type">>], - ItemVals, - [<<"username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]). +decode_subscription(<<"B">>) -> both; +decode_subscription(<<"T">>) -> to; +decode_subscription(<<"F">>) -> from; +decode_subscription(_) -> none. + +encode_subscription(both) -> <<"B">>; +encode_subscription(to) -> <<"T">>; +encode_subscription(from) -> <<"F">>; +encode_subscription(none) -> <<"N">>. + +decode_ask(<<"S">>) -> subscribe; +decode_ask(<<"U">>) -> unsubscribe; +decode_ask(<<"B">>) -> both; +decode_ask(<<"O">>) -> out; +decode_ask(<<"I">>) -> in; +decode_ask(_) -> none. + +encode_ask(subscribe) -> <<"S">>; +encode_ask(unsubscribe) -> <<"U">>; +encode_ask(both) -> <<"B">>; +encode_ask(out) -> <<"O">>; +encode_ask(in) -> <<"I">>; +encode_ask(none) -> <<"N">>. + +record_to_row(#roster{us = {LUser, _Server}, + jid = JID, name = Nick, subscription = Subscription, + ask = Ask, askmessage = AskMessage}) -> + BinJID = jid:to_binary(jid:to_lower(JID)), + ExtSubscription = encode_subscription(Subscription), + ExtAsk = encode_ask(Ask), + [LUser, BinJID, Nick, ExtSubscription, ExtAsk, AskMessage, + <<"N">>, <<>>, <<"item">>]. + +groups_to_rows(#roster{us = {LUser, _LServer}, jid = JID, groups = Groups}) -> + BinJID = jid:to_binary(jid:to_lower(JID)), + lists:foldl(fun (<<>>, Acc) -> Acc; + (Group, Acc) -> [[LUser, BinJID, Group] | Acc] + end, [], Groups). -q_get_subscription(Username, SJID) -> - [<<"select subscription from rosterusers " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), <<" " - "and jid=">>, mongoose_rdbms:use_escaped_string(SJID)]. +raw_to_record(LServer, + {User, SJID, Nick, ExtSubscription, ExtAsk, AskMessage, + _Server, _ExtSubscribe, _Type}) -> + case jid:from_binary(SJID) of + error -> error; + JID -> + LJID = jid:to_lower(JID), + Subscription = decode_subscription(ExtSubscription), + Ask = decode_ask(ExtAsk), + #roster{usj = {User, LServer, LJID}, + us = {User, LServer}, jid = LJID, name = Nick, + subscription = Subscription, ask = Ask, + askmessage = AskMessage} + end. -get_subscription(LServer, Username, SJID) -> - mongoose_rdbms:sql_query( LServer, q_get_subscription(Username, SJID)). +decode_roster_rows(LServer, Rows, JIDGroups) -> + GroupsDict = pairs_to_dict(JIDGroups), + F = fun (Row) -> raw_to_record_with_group(LServer, Row, GroupsDict) end, + lists:flatmap(F, Rows). -get_subscription_t(_LServer, Username, SJID) -> - mongoose_rdbms:sql_query_t(q_get_subscription(Username, SJID)). +pairs_to_dict(Pairs) -> + F = fun ({K, V}, Acc) -> dict:append(K, V, Acc) end, + lists:foldl(F, dict:new(), Pairs). +raw_to_record_with_group(LServer, I, GroupsDict) -> + case raw_to_record(LServer, I) of + %% Bad JID in database: + error -> []; + R -> + BinJID = jid:to_binary(R#roster.jid), + [R#roster{groups = dict_find_list(BinJID, GroupsDict)}] + end. + +dict_find_list(K, Dict) -> + case dict:find(K, Dict) of + {ok, Values} -> Values; + error -> [] + end. From d759e817b1ab9626f2510fd8da755368f50cca25 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 4 Jan 2021 23:39:13 +0100 Subject: [PATCH 62/92] Properly decode subscription and ask fields in mod_roster_rdbms --- src/mod_roster_rdbms.erl | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/mod_roster_rdbms.erl b/src/mod_roster_rdbms.erl index 34fee4522b..89b75ad17d 100644 --- a/src/mod_roster_rdbms.erl +++ b/src/mod_roster_rdbms.erl @@ -13,6 +13,7 @@ -include("mod_roster.hrl"). -include("jlib.hrl"). -include("mongoose.hrl"). +-include("mongoose_logger.hrl"). -behaviour(mod_roster). @@ -154,6 +155,7 @@ decode_roster_entry_rows(LUser, LServer, LJID, [Row]) -> case Rec of %% Bad JID in database: error -> + ?LOG_ERROR(#{what => roster_parse_failed, row => io_lib:format("~0p", [Row])}), #roster{usj = USJ, us = US, jid = LJID}; _ -> Rec#roster{usj = USJ, us = US, jid = LJID} @@ -176,7 +178,8 @@ get_roster_entry_t(LUser, LServer, LJID, full) -> get_subscription_lists(_, LUser, LServer) -> try execute_roster_get(LServer, LUser) of {selected, Rows} -> - Rows; + Rows; %% The only allowed usage of these rows is to pass them + %% into mod_roster_backend:raw_to_record/2 Other -> ?LOG_ERROR(#{what => get_subscription_lists_failed, reason => Other, user => LUser, host => LServer}), @@ -243,22 +246,26 @@ read_subscription_and_groups(LUser, LServer, LJID) -> %% Helper functions %%============================================================================== -decode_subscription(<<"B">>) -> both; -decode_subscription(<<"T">>) -> to; -decode_subscription(<<"F">>) -> from; -decode_subscription(_) -> none. +%% PgSQL returns CHAR as an integer +%% MySQL returns CHAR as a string +decode_subscription(<>) -> decode_subscription(X); +decode_subscription($B) -> both; +decode_subscription($T) -> to; +decode_subscription($F) -> from; +decode_subscription($N) -> none. encode_subscription(both) -> <<"B">>; encode_subscription(to) -> <<"T">>; encode_subscription(from) -> <<"F">>; encode_subscription(none) -> <<"N">>. -decode_ask(<<"S">>) -> subscribe; -decode_ask(<<"U">>) -> unsubscribe; -decode_ask(<<"B">>) -> both; -decode_ask(<<"O">>) -> out; -decode_ask(<<"I">>) -> in; -decode_ask(_) -> none. +decode_ask(<>) -> decode_ask(X); +decode_ask($S) -> subscribe; +decode_ask($U) -> unsubscribe; +decode_ask($B) -> both; +decode_ask($O) -> out; +decode_ask($I) -> in; +decode_ask($N) -> none. encode_ask(subscribe) -> <<"S">>; encode_ask(unsubscribe) -> <<"U">>; @@ -284,9 +291,11 @@ groups_to_rows(#roster{us = {LUser, _LServer}, jid = JID, groups = Groups}) -> raw_to_record(LServer, {User, SJID, Nick, ExtSubscription, ExtAsk, AskMessage, - _Server, _ExtSubscribe, _Type}) -> + _Server, _ExtSubscribe, _Type} = I) -> case jid:from_binary(SJID) of - error -> error; + error -> + ?LOG_ERROR(#{what => roster_parse_failed, row => io_lib:format("~0p", [I])}), + error; JID -> LJID = jid:to_lower(JID), Subscription = decode_subscription(ExtSubscription), From 3ef8057aa925c38d128cc9317cf8303fcba14757 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 7 Jan 2021 11:55:49 +0100 Subject: [PATCH 63/92] Remove unused roster_sub_get_by_jid query --- src/mod_roster_rdbms.erl | 56 +++++++++++------------------------- src/rdbms/mongoose_rdbms.erl | 8 ++++++ 2 files changed, 24 insertions(+), 40 deletions(-) diff --git a/src/mod_roster_rdbms.erl b/src/mod_roster_rdbms.erl index 89b75ad17d..4510dde3e5 100644 --- a/src/mod_roster_rdbms.erl +++ b/src/mod_roster_rdbms.erl @@ -56,9 +56,6 @@ prepare_queries(Host) -> " FROM rosterusers WHERE username=? AND jid=?">>), mongoose_rdbms:prepare(roster_group_get, rostergroups, [username], <<"SELECT jid, grp FROM rostergroups WHERE username=?">>), - mongoose_rdbms:prepare(roster_sub_get_by_jid, rosterusers, [username, jid], - <<"SELECT subscription FROM rosterusers " - "WHERE username=? AND jid=?">>), mongoose_rdbms:prepare(roster_group_get_by_jid, rostergroups, [username, jid], <<"SELECT grp FROM rostergroups " "WHERE username=? AND jid=?">>), @@ -155,21 +152,19 @@ decode_roster_entry_rows(LUser, LServer, LJID, [Row]) -> case Rec of %% Bad JID in database: error -> - ?LOG_ERROR(#{what => roster_parse_failed, row => io_lib:format("~0p", [Row])}), + ?LOG_ERROR(#{what => roster_parse_failed, row => format_term(Row)}), #roster{usj = USJ, us = US, jid = LJID}; _ -> Rec#roster{usj = USJ, us = US, jid = LJID} end. +%% full means we should query for groups too get_roster_entry(LUser, LServer, LJID, full) -> case get_roster_entry(LUser, LServer, LJID) of does_not_exist -> does_not_exist; - Rentry -> - case read_subscription_and_groups(LUser, LServer, LJID) of - error -> error; - {Subscription, Groups} -> - Rentry#roster{subscription = Subscription, groups = Groups} - end + Rec -> + Groups = get_groups_by_jid(LUser, LServer, LJID), + Rec#roster{groups = Groups} end. get_roster_entry_t(LUser, LServer, LJID, full) -> @@ -219,36 +214,16 @@ del_roster_t(LUser, LServer, LJID) -> mongoose_rdbms:execute(LServer, roster_delete_by_jid, [LUser, BinJID]), mongoose_rdbms:execute(LServer, roster_group_delete_by_jid, [LUser, BinJID]). -read_subscription_and_groups(LUser, LServer, LJID) -> +get_groups_by_jid(LUser, LServer, LJID) -> BinJID = jid:to_binary(LJID), - SubResult = mongoose_rdbms:execute(LServer, roster_sub_get_by_jid, [LUser, BinJID]), - case SubResult of - {selected, [{ExtSubscription}]} -> - GroupResult = mongoose_rdbms:execute(LServer, roster_group_get_by_jid, [LUser, BinJID]), - Subscription = decode_subscription(ExtSubscription), - case GroupResult of - {selected, GroupRows} -> - Groups = [JGrp || {JGrp} <- GroupRows], - {Subscription, Groups}; - _ -> - ?LOG_ERROR(#{what => read_subscription_and_groups_failed, - reason => GroupResult, user => LUser, - host => LServer}), - error - end; - E -> - ?LOG_ERROR(#{what => read_subscription_and_groups_failed, - reason => E, user => LUser, host => LServer}), - error - end. + {selected, Rows} = mongoose_rdbms:execute_successfully( + LServer, roster_group_get_by_jid, [LUser, BinJID]), + [Group || {Group} <- Rows]. %%============================================================================== %% Helper functions %%============================================================================== -%% PgSQL returns CHAR as an integer -%% MySQL returns CHAR as a string -decode_subscription(<>) -> decode_subscription(X); decode_subscription($B) -> both; decode_subscription($T) -> to; decode_subscription($F) -> from; @@ -259,7 +234,6 @@ encode_subscription(to) -> <<"T">>; encode_subscription(from) -> <<"F">>; encode_subscription(none) -> <<"N">>. -decode_ask(<>) -> decode_ask(X); decode_ask($S) -> subscribe; decode_ask($U) -> unsubscribe; decode_ask($B) -> both; @@ -298,8 +272,8 @@ raw_to_record(LServer, error; JID -> LJID = jid:to_lower(JID), - Subscription = decode_subscription(ExtSubscription), - Ask = decode_ask(ExtAsk), + Subscription = decode_subscription(mongoose_rdbms:character_to_integer(ExtSubscription)), + Ask = decode_ask(mongoose_rdbms:character_to_integer(ExtAsk)), #roster{usj = {User, LServer, LJID}, us = {User, LServer}, jid = LJID, name = Nick, subscription = Subscription, ask = Ask, @@ -321,11 +295,13 @@ raw_to_record_with_group(LServer, I, GroupsDict) -> error -> []; R -> BinJID = jid:to_binary(R#roster.jid), - [R#roster{groups = dict_find_list(BinJID, GroupsDict)}] + [R#roster{groups = dict_find(BinJID, GroupsDict, [])}] end. -dict_find_list(K, Dict) -> +dict_find(K, Dict, Default) -> case dict:find(K, Dict) of {ok, Values} -> Values; - error -> [] + error -> Default end. + +format_term(X) -> iolist_to_binary(io_lib:format("~0p", [X])). diff --git a/src/rdbms/mongoose_rdbms.erl b/src/rdbms/mongoose_rdbms.erl index b3a33ede95..7bcca2e96f 100644 --- a/src/rdbms/mongoose_rdbms.erl +++ b/src/rdbms/mongoose_rdbms.erl @@ -120,6 +120,8 @@ -export([result_to_integer/1, selected_to_integer/1]). +-export([character_to_integer/1]). + %% gen_server callbacks -export([init/1, handle_call/3, @@ -477,6 +479,12 @@ result_to_integer(Bin) when is_binary(Bin) -> selected_to_integer({selected, [{BInt}]}) -> result_to_integer(BInt). +%% Converts a value from a CHAR field to integer +%% PgSQL returns CHAR as an integer +%% %% MySQL returns CHAR as a string +character_to_integer(<>) -> X; +character_to_integer(X) when is_integer(X) -> X. + %% pgsql returns booleans as "t" or "f" -spec to_bool(binary() | string() | atom() | integer() | any()) -> boolean(). to_bool(B) when is_binary(B) -> From 11bf2a01d8629ed9edcfdc788bfe5506abd5e0d6 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 7 Jan 2021 12:35:45 +0100 Subject: [PATCH 64/92] Simplify error handling in mod_roster_rdbms Require database jid to be parsable --- priv/mysql.sql | 2 +- src/mod_roster_rdbms.erl | 138 ++++++++++++++++----------------------- 2 files changed, 59 insertions(+), 81 deletions(-) diff --git a/priv/mysql.sql b/priv/mysql.sql index c9efcc9ae0..046ba176f7 100644 --- a/priv/mysql.sql +++ b/priv/mysql.sql @@ -58,7 +58,7 @@ CREATE INDEX i_last_seconds ON last(seconds); CREATE TABLE rosterusers ( username varchar(250) NOT NULL, - jid varchar(250) NOT NULL, + jid varchar(250) NOT NULL, -- must be a parsable jid nick text NOT NULL, subscription character(1) NOT NULL, ask character(1) NOT NULL, diff --git a/src/mod_roster_rdbms.erl b/src/mod_roster_rdbms.erl index 4510dde3e5..b11d80d83d 100644 --- a/src/mod_roster_rdbms.erl +++ b/src/mod_roster_rdbms.erl @@ -71,9 +71,10 @@ prepare_queries(Host) -> prepare_version_upsert(Host), ok. + +%% We don't care about `server, subscribe, type' fields roster_fields() -> - <<"username, jid, nick, subscription, ask, " - "askmessage, server, subscribe, type">>. + <<"username, jid, nick, subscription, ask, askmessage">>. prepare_roster_upsert(Host) -> Fields = [<<"nick">>, <<"subscription">>, <<"ask">>, @@ -91,7 +92,7 @@ prepare_version_upsert(Host) -> %% Query Helpers execute_roster_get(LServer, LUser) -> - mongoose_rdbms:execute(LServer, roster_get, [LUser]). + mongoose_rdbms:execute_successfully(LServer, roster_get, [LUser]). roster_upsert(Host, LUser, BinJID, RosterRow) -> [_LUser, _BinJID|Rest] = RosterRow, @@ -115,7 +116,7 @@ transaction(LServer, F) -> -spec read_roster_version(jid:luser(), jid:lserver()) -> binary() | error. read_roster_version(LUser, LServer) -> - case mongoose_rdbms:execute(LServer, roster_version_get, [LUser]) of + case mongoose_rdbms:execute_successfully(LServer, roster_version_get, [LUser]) of {selected, [{Version}]} -> Version; {selected, []} -> error end. @@ -124,67 +125,37 @@ write_roster_version(LUser, LServer, _InTransaction, Ver) -> version_upsert(LServer, LUser, Ver). get_roster(LUser, LServer) -> - try execute_roster_get(LServer, LUser) of - {selected, Rows} -> - {selected, GroupRows} = mongoose_rdbms:execute(LServer, roster_group_get, [LUser]), - decode_roster_rows(LServer, Rows, GroupRows); - _ -> [] - catch Class:Reason:StackTrace -> - ?LOG_ERROR(#{what => get_roster_failed, class => Class, reason => Reason, - stacktrace => StackTrace, user => LUser, host => LServer}), - [] - end. + {selected, Rows} = execute_roster_get(LServer, LUser), + {selected, GroupRows} = mongoose_rdbms:execute_successfully(LServer, roster_group_get, [LUser]), + decode_roster_rows(LServer, Rows, GroupRows). get_roster_entry_t(LUser, LServer, LJID) -> + %% `mongoose_rdbms:execute' automatically detects if we are in a transaction or not. get_roster_entry(LUser, LServer, LJID). get_roster_entry(LUser, LServer, LJID) -> BinJID = jid:to_binary(LJID), - {selected, Rows} = mongoose_rdbms:execute(LServer, roster_get_by_jid, [LUser, BinJID]), - decode_roster_entry_rows(LUser, LServer, LJID, Rows). - -decode_roster_entry_rows(_LUser, _LServer, _LJID, []) -> - does_not_exist; -decode_roster_entry_rows(LUser, LServer, LJID, [Row]) -> - Rec = raw_to_record(LServer, Row), - USJ = {LUser, LServer, LJID}, - US = {LUser, LServer}, - case Rec of - %% Bad JID in database: - error -> - ?LOG_ERROR(#{what => roster_parse_failed, row => format_term(Row)}), - #roster{usj = USJ, us = US, jid = LJID}; - _ -> - Rec#roster{usj = USJ, us = US, jid = LJID} + {selected, Rows} = mongoose_rdbms:execute_successfully(LServer, roster_get_by_jid, [LUser, BinJID]), + case Rows of + [] -> does_not_exist; + [Row] -> row_to_record(LServer, Row) end. +get_roster_entry_t(LUser, LServer, LJID, full) -> + get_roster_entry(LUser, LServer, LJID, full). + %% full means we should query for groups too get_roster_entry(LUser, LServer, LJID, full) -> case get_roster_entry(LUser, LServer, LJID) of does_not_exist -> does_not_exist; Rec -> Groups = get_groups_by_jid(LUser, LServer, LJID), - Rec#roster{groups = Groups} + record_with_groups(Rec, Groups) end. -get_roster_entry_t(LUser, LServer, LJID, full) -> - get_roster_entry(LUser, LServer, LJID, full). - get_subscription_lists(_, LUser, LServer) -> - try execute_roster_get(LServer, LUser) of - {selected, Rows} -> - Rows; %% The only allowed usage of these rows is to pass them - %% into mod_roster_backend:raw_to_record/2 - Other -> - ?LOG_ERROR(#{what => get_subscription_lists_failed, reason => Other, - user => LUser, host => LServer}), - [] - catch Class:Reason:StackTrace -> - ?LOG_ERROR(#{what => get_subscription_lists_failed, class => Class, - reason => Reason, stacktrace => StackTrace, - user => LUser, host => LServer}), - [] - end. + {selected, Rows} = execute_roster_get(LServer, LUser), + [row_to_record(LServer, Row) || Row <- Rows]. roster_subscribe_t(LUser, LServer, LJID, Item) -> BinJID = jid:to_binary(LJID), @@ -193,8 +164,8 @@ roster_subscribe_t(LUser, LServer, LJID, Item) -> remove_user(LUser, LServer) -> F = fun() -> - mongoose_rdbms:execute(LServer, roster_delete, [LUser]), - mongoose_rdbms:execute(LServer, roster_group_delete, [LUser]) + mongoose_rdbms:execute_successfully(LServer, roster_delete, [LUser]), + mongoose_rdbms:execute_successfully(LServer, roster_group_delete, [LUser]) end, mongoose_rdbms:sql_transaction(LServer, F), ok. @@ -204,15 +175,15 @@ update_roster_t(LUser, LServer, LJID, Item) -> RosterRow = record_to_row(Item), GroupRows = groups_to_rows(Item), roster_upsert(LServer, LUser, BinJID, RosterRow), - mongoose_rdbms:execute(LServer, roster_group_delete_by_jid, [LUser, BinJID]), - [mongoose_rdbms:execute(LServer, roster_group_insert, GroupRow) + mongoose_rdbms:execute_successfully(LServer, roster_group_delete_by_jid, [LUser, BinJID]), + [mongoose_rdbms:execute_successfully(LServer, roster_group_insert, GroupRow) || GroupRow <- GroupRows], ok. del_roster_t(LUser, LServer, LJID) -> BinJID = jid:to_binary(LJID), - mongoose_rdbms:execute(LServer, roster_delete_by_jid, [LUser, BinJID]), - mongoose_rdbms:execute(LServer, roster_group_delete_by_jid, [LUser, BinJID]). + mongoose_rdbms:execute_successfully(LServer, roster_delete_by_jid, [LUser, BinJID]), + mongoose_rdbms:execute_successfully(LServer, roster_group_delete_by_jid, [LUser, BinJID]). get_groups_by_jid(LUser, LServer, LJID) -> BinJID = jid:to_binary(LJID), @@ -263,45 +234,52 @@ groups_to_rows(#roster{us = {LUser, _LServer}, jid = JID, groups = Groups}) -> (Group, Acc) -> [[LUser, BinJID, Group] | Acc] end, [], Groups). -raw_to_record(LServer, - {User, SJID, Nick, ExtSubscription, ExtAsk, AskMessage, - _Server, _ExtSubscribe, _Type} = I) -> - case jid:from_binary(SJID) of +%% We must not leak our external RDBMS format representation into MongooseIM +raw_to_record(_LServer, Rec) -> Rec. + +%% Decode fields from `roster_fields()' into a record +row_to_record(LServer, + {User, BinJID, Nick, ExtSubscription, ExtAsk, AskMessage}) -> + JID = parse_jid(BinJID), + LJID = jid:to_lower(JID), %% Convert to tuple {U,S,R} + Subscription = decode_subscription(mongoose_rdbms:character_to_integer(ExtSubscription)), + Ask = decode_ask(mongoose_rdbms:character_to_integer(ExtAsk)), + USJ = {User, LServer, LJID}, + US = {User, LServer}, + #roster{usj = USJ, us = US, jid = LJID, name = Nick, + subscription = Subscription, ask = Ask, askmessage = AskMessage}. + +row_to_binary_jid(Row) -> element(2, Row). + +record_with_groups(Rec, Groups) -> + Rec#roster{groups = Groups}. + +%% We require all DB jids to be parsable. +%% They should be lowered too. +parse_jid(BinJID) -> + case jid:from_binary(BinJID) of error -> - ?LOG_ERROR(#{what => roster_parse_failed, row => io_lib:format("~0p", [I])}), - error; + error(#{what => parse_jid_failed, jid => BinJID}); JID -> - LJID = jid:to_lower(JID), - Subscription = decode_subscription(mongoose_rdbms:character_to_integer(ExtSubscription)), - Ask = decode_ask(mongoose_rdbms:character_to_integer(ExtAsk)), - #roster{usj = {User, LServer, LJID}, - us = {User, LServer}, jid = LJID, name = Nick, - subscription = Subscription, ask = Ask, - askmessage = AskMessage} + JID end. decode_roster_rows(LServer, Rows, JIDGroups) -> GroupsDict = pairs_to_dict(JIDGroups), - F = fun (Row) -> raw_to_record_with_group(LServer, Row, GroupsDict) end, - lists:flatmap(F, Rows). + [raw_to_record_with_group(LServer, Row, GroupsDict) || Row <- Rows]. pairs_to_dict(Pairs) -> F = fun ({K, V}, Acc) -> dict:append(K, V, Acc) end, lists:foldl(F, dict:new(), Pairs). -raw_to_record_with_group(LServer, I, GroupsDict) -> - case raw_to_record(LServer, I) of - %% Bad JID in database: - error -> []; - R -> - BinJID = jid:to_binary(R#roster.jid), - [R#roster{groups = dict_find(BinJID, GroupsDict, [])}] - end. +raw_to_record_with_group(LServer, Row, GroupsDict) -> + Rec = row_to_record(LServer, Row), + BinJID = row_to_binary_jid(Row), + Groups = dict_get(BinJID, GroupsDict, []), + record_with_groups(Rec, Groups). -dict_find(K, Dict, Default) -> +dict_get(K, Dict, Default) -> case dict:find(K, Dict) of {ok, Values} -> Values; error -> Default end. - -format_term(X) -> iolist_to_binary(io_lib:format("~0p", [X])). From 5274208664b354f356f78be946e9d6dbbcfb1cb6 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 8 Jan 2021 10:47:50 +0100 Subject: [PATCH 65/92] Edit comment for character_to_integer --- src/rdbms/mongoose_rdbms.erl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/rdbms/mongoose_rdbms.erl b/src/rdbms/mongoose_rdbms.erl index 7bcca2e96f..156068f7d6 100644 --- a/src/rdbms/mongoose_rdbms.erl +++ b/src/rdbms/mongoose_rdbms.erl @@ -479,9 +479,7 @@ result_to_integer(Bin) when is_binary(Bin) -> selected_to_integer({selected, [{BInt}]}) -> result_to_integer(BInt). -%% Converts a value from a CHAR field to integer -%% PgSQL returns CHAR as an integer -%% %% MySQL returns CHAR as a string +%% Converts a value from a CHAR(1) field to integer character_to_integer(<>) -> X; character_to_integer(X) when is_integer(X) -> X. From f761742c3c3a5f176b00d035393ca63d7ce1774c Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 12 Jan 2021 13:33:05 +0100 Subject: [PATCH 66/92] Use prepared queries in mod_privacy_rdbms --- src/mod_privacy_rdbms.erl | 361 ++++++++++++++++++------------------ src/rdbms/rdbms_queries.erl | 131 ------------- 2 files changed, 183 insertions(+), 309 deletions(-) diff --git a/src/mod_privacy_rdbms.erl b/src/mod_privacy_rdbms.erl index 03ed91408c..1b2f8c0932 100644 --- a/src/mod_privacy_rdbms.erl +++ b/src/mod_privacy_rdbms.erl @@ -44,9 +44,62 @@ -include("jlib.hrl"). -include("mod_privacy.hrl"). -init(_Host, _Opts) -> +init(Host, _Opts) -> + prepare_queries(Host), ok. +prepare_queries(Host) -> + %% Queries to privacy_list table + mongoose_rdbms:prepare(privacy_list_get_id, privacy_list, [username, name], + <<"SELECT id FROM privacy_list where username=? AND name=?">>), + mongoose_rdbms:prepare(privacy_list_get_names, privacy_list, [username], + <<"SELECT name FROM privacy_list WHERE username=?">>), + mongoose_rdbms:prepare(privacy_list_delete_by_name, privacy_list, [username, name], + <<"DELETE FROM privacy_list WHERE username=? AND name=?">>), + mongoose_rdbms:prepare(privacy_list_delete_multiple, privacy_list, [username], + <<"DELETE FROM privacy_list WHERE username=?">>), + mongoose_rdbms:prepare(privacy_list_insert, privacy_list, [username, name], + <<"INSERT INTO privacy_list(username, name) VALUES (?, ?)">>), + %% Queries to privacy_default_list table + mongoose_rdbms:prepare(privacy_default_get_name, privacy_default_list, [username], + <<"SELECT name FROM privacy_default_list WHERE username=?">>), + mongoose_rdbms:prepare(privacy_default_delete, privacy_default_list, [username], + <<"DELETE from privacy_default_list WHERE username=?">>), + prepare_default_list_upsert(Host), + %% Queries to privacy_list_data table + mongoose_rdbms:prepare(privacy_data_get_by_id, privacy_list_data, [id], + <<"SELECT t, value, action, ord, match_all, match_iq, " + "match_message, match_presence_in, match_presence_out " + "FROM privacy_list_data " + "WHERE id=? ORDER BY ord">>), + mongoose_rdbms:prepare(delete_data_by_id, privacy_list_data, [id], + <<"DELETE FROM privacy_list_data WHERE id=?">>), + mongoose_rdbms:prepare(privacy_data_insert, privacy_list_data, + [id, t, value, action, ord, match_all, match_iq, + match_message, match_presence_in, match_presence_out], + <<"INSERT INTO privacy_list_data(" + "id, t, value, action, ord, match_all, match_iq, " + "match_message, match_presence_in, match_presence_out) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)">>), + %% This query uses multiple tables + mongoose_rdbms:prepare(privacy_data_delete_user, privacy_list, [username], + <<"DELETE FROM privacy_list_data WHERE id IN " + "(SELECT id FROM privacy_list WHERE username=?)">>), + ok. + +prepare_default_list_upsert(Host) -> + Fields = [<<"name">>], + Filter = [<<"username">>], + rdbms_queries:prepare_upsert(Host, privacy_default_upsert, privacy_default_list, + Filter ++ Fields, Fields, Filter). + +default_list_upsert(Host, LUser, Name) -> + InsertParams = [LUser, Name], + UpdateParams = [Name], + UniqueKeyValues = [LUser], + rdbms_queries:execute_upsert(Host, privacy_default_upsert, + InsertParams, UpdateParams, UniqueKeyValues). + get_default_list(LUser, LServer) -> case get_default_list_name(LUser, LServer) of none -> @@ -66,7 +119,7 @@ get_list_names(LUser, LServer) -> {ok, {Default, Names}}. get_default_list_name(LUser, LServer) -> - try sql_get_default_privacy_list(LUser, LServer) of + try execute_privacy_default_get_name(LServer, LUser) of {selected, []} -> none; {selected, [{DefName}]} -> @@ -84,7 +137,7 @@ get_default_list_name(LUser, LServer) -> end. get_list_names_only(LUser, LServer) -> - try sql_get_privacy_list_names(LUser, LServer) of + try execute_privacy_list_get_names(LServer, LUser) of {selected, Names} -> [Name || {Name} <- Names]; Other -> @@ -99,9 +152,8 @@ get_list_names_only(LUser, LServer) -> [] end. - get_privacy_list(LUser, LServer, Name) -> - try sql_get_privacy_list_id(LUser, LServer, Name) of + try execute_privacy_list_get_id(LServer, LUser, Name) of {selected, []} -> {error, not_found}; {selected, [{ID}]} -> @@ -121,10 +173,9 @@ get_privacy_list(LUser, LServer, Name) -> end. get_privacy_list_by_id(LUser, LServer, Name, ID, LServer) when is_integer(ID) -> - try sql_get_privacy_list_data_by_id(ID, LServer) of - {selected, RItems} -> - Items = raw_to_items(RItems), - {ok, Items}; + try execute_privacy_data_get_by_id(LServer, ID) of + {selected, Rows} -> + {ok, raw_to_items(Rows)}; Other -> ?LOG_ERROR(#{what => privacy_get_privacy_list_by_id_failed, user => LUser, server => LServer, list_name => Name, list_id => ID, @@ -138,12 +189,9 @@ get_privacy_list_by_id(LUser, LServer, Name, ID, LServer) when is_integer(ID) -> {error, Reason} end. -raw_to_items(RItems) -> - lists:map(fun raw_to_item/1, RItems). - %% @doc Set no default list for user. forget_default_list(LUser, LServer) -> - try sql_unset_default_privacy_list(LUser, LServer) of + try execute_privacy_default_delete(LServer, LUser) of {updated, _} -> ok; Other -> @@ -159,8 +207,8 @@ forget_default_list(LUser, LServer) -> end. set_default_list(LUser, LServer, Name) -> - case rdbms_queries:sql_transaction( - LServer, fun() -> set_default_list_t(LUser, Name) end) of + F = fun() -> set_default_list_t(LServer, LUser, Name) end, + case rdbms_queries:sql_transaction(LServer, F) of {atomic, ok} -> ok; {atomic, {error, Reason}} -> @@ -171,15 +219,14 @@ set_default_list(LUser, LServer, Name) -> {error, Reason} end. --spec set_default_list_t(LUser :: jid:luser(), Name :: binary()) -> ok | {error, not_found}. -set_default_list_t(LUser, Name) -> - case sql_get_privacy_list_names_t(LUser) of +set_default_list_t(LServer, LUser, Name) -> + case execute_privacy_list_get_names(LServer, LUser) of {selected, []} -> {error, not_found}; {selected, Names} -> case lists:member({Name}, Names) of true -> - sql_set_default_privacy_list(LUser, Name), + default_list_upsert(LServer, LUser, Name), ok; false -> {error, not_found} @@ -188,14 +235,12 @@ set_default_list_t(LUser, Name) -> remove_privacy_list(LUser, LServer, Name) -> F = fun() -> - case sql_get_default_privacy_list_t(LUser) of - {selected, []} -> - sql_remove_privacy_list(LUser, Name), - ok; - {selected, [{Name}]} -> + case execute_privacy_default_get_name(LServer, LUser) of + {selected, [{Name}]} -> %% Matches Name variable {error, conflict}; - {selected, [{_Default}]} -> - sql_remove_privacy_list(LUser, Name) + {selected, _} -> + execute_privacy_list_delete_by_name(LServer, LUser, Name), + ok end end, case rdbms_queries:sql_transaction(LServer, F) of @@ -210,18 +255,18 @@ remove_privacy_list(LUser, LServer, Name) -> end. replace_privacy_list(LUser, LServer, Name, List) -> - RItems = lists:map(fun item_to_raw/1, List), + Rows = lists:map(fun item_to_raw/1, List), F = fun() -> - ID = case sql_get_privacy_list_id_t(LUser, Name) of + ResultID = case execute_privacy_list_get_id(LServer, LUser, Name) of {selected, []} -> - sql_add_privacy_list(LUser, Name), - {selected, [{I}]} = - sql_get_privacy_list_id_t(LUser, Name), + execute_privacy_list_insert(LServer, LUser, Name), + {selected, [{I}]} = execute_privacy_list_get_id(LServer, LUser, Name), I; {selected, [{I}]} -> I end, - sql_set_privacy_list(mongoose_rdbms:result_to_integer(ID), RItems), + ID = mongoose_rdbms:result_to_integer(ResultID), + replace_data_rows(LServer, ID, Rows), ok end, case rdbms_queries:sql_transaction(LServer, F) of @@ -234,58 +279,65 @@ replace_privacy_list(LUser, LServer, Name, List) -> end. remove_user(LUser, LServer) -> - sql_del_privacy_lists(LUser, LServer). - - -raw_to_item({BType, BValue, BAction, BOrder, BMatchAll, BMatchIQ, - BMatchMessage, BMatchPresenceIn, BMatchPresenceOut}) -> - {Type, Value} = - case BType of - <<"n">> -> - {none, none}; - <<"j">> -> - case jid:from_binary(BValue) of - #jid{} = JID -> - {jid, jid:to_lower(JID)} - end; - <<"g">> -> - {group, BValue}; - <<"s">> -> - case BValue of - <<"none">> -> - {subscription, none}; - <<"both">> -> - {subscription, both}; - <<"from">> -> - {subscription, from}; - <<"to">> -> - {subscription, to} - end - end, - Action = - case BAction of - <<"a">> -> allow; - <<"d">> -> deny; - <<"b">> -> block - end, - Order = mongoose_rdbms:result_to_integer(BOrder), - MatchAll = mongoose_rdbms:to_bool(BMatchAll), - MatchIQ = mongoose_rdbms:to_bool(BMatchIQ), - MatchMessage = mongoose_rdbms:to_bool(BMatchMessage), - MatchPresenceIn = mongoose_rdbms:to_bool(BMatchPresenceIn), - MatchPresenceOut = mongoose_rdbms:to_bool(BMatchPresenceOut), + F = fun() -> remove_user_t(LUser, LServer) end, + rdbms_queries:sql_transaction(LServer, F). + +remove_user_t(LUser, LServer) -> + mongoose_rdbms:execute_successfully(LServer, privacy_data_delete_user, [LUser]), + mongoose_rdbms:execute_successfully(LServer, privacy_list_delete_multiple, [LUser]), + mongoose_rdbms:execute_successfully(LServer, privacy_default_delete, [LUser]). + +execute_privacy_list_get_id(LServer, LUser, Name) -> + mongoose_rdbms:execute_successfully(LServer, privacy_list_get_id, [LUser, Name]). + +execute_privacy_default_get_name(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, privacy_default_get_name, [LUser]). + +execute_privacy_list_get_names(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, privacy_list_get_names, [LUser]). + +execute_privacy_data_get_by_id(LServer, ID) -> + mongoose_rdbms:execute_successfully(LServer, privacy_data_get_by_id, [ID]). + +execute_privacy_default_delete(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, privacy_default_delete, [LUser]). + +execute_privacy_list_delete_by_name(LServer, LUser, Name) -> + mongoose_rdbms:execute_successfully(LServer, privacy_list_delete_by_name, [LUser, Name]). + +execute_privacy_list_insert(LServer, LUser, Name) -> + mongoose_rdbms:execute_successfully(LServer, privacy_list_insert, [LUser, Name]). + +execute_delete_data_by_id(LServer, ID) -> + mongoose_rdbms:execute_successfully(LServer, delete_data_by_id, [ID]). + +replace_data_rows(LServer, ID, Rows) when is_integer(ID)-> + execute_delete_data_by_id(LServer, ID), + [mongoose_rdbms:execute_successfully(LServer, privacy_data_insert, [ID|Args]) + || Args <- Rows], + ok. + +%% Encoding/decoding pure functions + +raw_to_items(Rows) -> + [raw_to_item(Row) || Row <- Rows]. + +raw_to_item({ExtType, ExtValue, ExtAction, ExtOrder, + ExtMatchAll, ExtMatchIQ, ExtMatchMessage, + ExtMatchPresenceIn, ExtMatchPresenceOut}) -> + Type = decode_type(mongoose_rdbms:character_to_integer(ExtType)), #listitem{type = Type, - value = Value, - action = Action, - order = Order, - match_all = MatchAll, - match_iq = MatchIQ, - match_message = MatchMessage, - match_presence_in = MatchPresenceIn, - match_presence_out = MatchPresenceOut - }. - --spec item_to_raw(mod_privacy:list_item()) -> list(mongoose_rdbms:escaped_value()). + value = decode_value(Type, ExtValue), + action = decode_action(mongoose_rdbms:character_to_integer(ExtAction)), + order = mongoose_rdbms:result_to_integer(ExtOrder), + match_all = mongoose_rdbms:to_bool(ExtMatchAll), + match_iq = mongoose_rdbms:to_bool(ExtMatchIQ), + match_message = mongoose_rdbms:to_bool(ExtMatchMessage), + match_presence_in = mongoose_rdbms:to_bool(ExtMatchPresenceIn), + match_presence_out = mongoose_rdbms:to_bool(ExtMatchPresenceOut)}. + +%% Encodes for privacy_data_insert query (but without ID) +-spec item_to_raw(mod_privacy:list_item()) -> list(term()). item_to_raw(#listitem{type = Type, value = Value, action = Action, @@ -294,98 +346,51 @@ item_to_raw(#listitem{type = Type, match_iq = MatchIQ, match_message = MatchMessage, match_presence_in = MatchPresenceIn, - match_presence_out = MatchPresenceOut - }) -> - {BType, BValue} = - case Type of - none -> - {<<"n">>, <<>>}; - jid -> - {<<"j">>, jid:to_binary(Value)}; - group -> - {<<"g">>, Value}; - subscription -> - case Value of - none -> - {<<"s">>, <<"none">>}; - both -> - {<<"s">>, <<"both">>}; - from -> - {<<"s">>, <<"from">>}; - to -> - {<<"s">>, <<"to">>} - end - end, - SType = mongoose_rdbms:escape_string(BType), - SValue = mongoose_rdbms:escape_string(BValue), - BAction = - case Action of - allow -> <<"a">>; - deny -> <<"d">>; - block -> <<"b">> - end, - SAction = mongoose_rdbms:escape_string(BAction), - SOrder = mongoose_rdbms:escape_integer(Order), - SMatchAll = mongoose_rdbms:escape_boolean(MatchAll), - SMatchIQ = mongoose_rdbms:escape_boolean(MatchIQ), - SMatchMessage = mongoose_rdbms:escape_boolean(MatchMessage), - SMatchPresenceIn = mongoose_rdbms:escape_boolean(MatchPresenceIn), - SMatchPresenceOut = mongoose_rdbms:escape_boolean(MatchPresenceOut), - [SType, SValue, SAction, SOrder, SMatchAll, SMatchIQ, - SMatchMessage, SMatchPresenceIn, SMatchPresenceOut]. - -sql_get_default_privacy_list(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:get_default_privacy_list(LServer, Username). - -sql_get_default_privacy_list_t(LUser) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:get_default_privacy_list_t(Username). - -sql_get_privacy_list_names(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:get_privacy_list_names(LServer, Username). - -sql_get_privacy_list_names_t(LUser) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:get_privacy_list_names_t(Username). - -sql_get_privacy_list_id(LUser, LServer, Name) -> - Username = mongoose_rdbms:escape_string(LUser), - SName = mongoose_rdbms:escape_string(Name), - rdbms_queries:get_privacy_list_id(LServer, Username, SName). - -sql_get_privacy_list_id_t(LUser, Name) -> - Username = mongoose_rdbms:escape_string(LUser), - SName = mongoose_rdbms:escape_string(Name), - rdbms_queries:get_privacy_list_id_t(Username, SName). - -sql_get_privacy_list_data_by_id(ID, LServer) when is_integer(ID) -> - rdbms_queries:get_privacy_list_data_by_id(LServer, mongoose_rdbms:escape_integer(ID)). - -sql_set_default_privacy_list(LUser, Name) -> - Username = mongoose_rdbms:escape_string(LUser), - SName = mongoose_rdbms:escape_string(Name), - rdbms_queries:set_default_privacy_list(Username, SName). - -sql_unset_default_privacy_list(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - rdbms_queries:unset_default_privacy_list(LServer, Username). - -sql_remove_privacy_list(LUser, Name) -> - Username = mongoose_rdbms:escape_string(LUser), - SName = mongoose_rdbms:escape_string(Name), - rdbms_queries:remove_privacy_list(Username, SName). - -sql_add_privacy_list(LUser, Name) -> - Username = mongoose_rdbms:escape_string(LUser), - SName = mongoose_rdbms:escape_string(Name), - rdbms_queries:add_privacy_list(Username, SName). - -sql_set_privacy_list(ID, RItems) when is_integer(ID)-> - rdbms_queries:set_privacy_list(mongoose_rdbms:escape_integer(ID), RItems). - -sql_del_privacy_lists(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - Server = mongoose_rdbms:escape_string(LServer), - rdbms_queries:del_privacy_lists(LServer, Server, Username). + match_presence_out = MatchPresenceOut}) -> + ExtType = encode_type(Type), + ExtValue = encode_value(Type, Value), + ExtAction = encode_action(Action), + Bools = [MatchAll, MatchIQ, MatchMessage, MatchPresenceIn, MatchPresenceOut], + ExtBools = [encode_boolean(X) || X <- Bools], + [ExtType, ExtValue, ExtAction, Order | ExtBools]. + +encode_boolean(true) -> 1; +encode_boolean(false) -> 0. + +encode_action(allow) -> <<"a">>; +encode_action(deny) -> <<"d">>; +encode_action(block) -> <<"b">>. + +decode_action($a) -> allow; +decode_action($d) -> deny; +decode_action($b) -> block. + +encode_subscription(none) -> <<"none">>; +encode_subscription(both) -> <<"both">>; +encode_subscription(from) -> <<"from">>; +encode_subscription(to) -> <<"to">>. + +decode_subscription(<<"none">>) -> none; +decode_subscription(<<"both">>) -> both; +decode_subscription(<<"from">>) -> from; +decode_subscription(<<"to">>) -> to. + +encode_type(none) -> <<"n">>; +encode_type(jid) -> <<"j">>; +encode_type(group) -> <<"g">>; +encode_type(subscription) -> <<"s">>. + +decode_type($n) -> none; +decode_type($j) -> jid; +decode_type($g) -> group; +decode_type($s) -> subscription. + +encode_value(none, _Value) -> <<>>; +encode_value(jid, Value) -> jid:to_binary(Value); +encode_value(group, Value) -> Value; +encode_value(subscription, Value) -> encode_subscription(Value). + +decode_value(none, _) -> none; +decode_value(jid, BinJid) -> jid:to_lower(jid:from_binary(BinJid)); +decode_value(group, Group) -> Group; +decode_value(subscription, ExtSub) -> decode_subscription(ExtSub). diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index 7603586bb2..55d2967f4a 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -50,22 +50,6 @@ users_number/2, get_users_without_scram/2, get_users_without_scram_count/1, - get_default_privacy_list/2, - get_default_privacy_list_t/1, - count_privacy_lists/1, - clear_privacy_lists/1, - get_privacy_list_names/2, - get_privacy_list_names_t/1, - get_privacy_list_id/3, - get_privacy_list_id_t/2, - get_privacy_list_data/3, - get_privacy_list_data_by_id/2, - set_default_privacy_list/2, - unset_default_privacy_list/2, - remove_privacy_list/2, - add_privacy_list/2, - set_privacy_list/2, - del_privacy_lists/3, count_records_where/3, prepare_offline_message/7, push_offline_messages/2, @@ -417,121 +401,6 @@ get_users_without_scram_count(LServer) -> LServer, [<<"select count(*) from users where pass_details is null">>]). -get_default_privacy_list(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select name from privacy_default_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_default_privacy_list_t(Username) -> - mongoose_rdbms:sql_query_t( - [<<"select name from privacy_default_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -count_privacy_lists(LServer) -> - mongoose_rdbms:sql_query(LServer, [<<"select count(*) from privacy_list;">>]). - -clear_privacy_lists(LServer) -> - mongoose_rdbms:sql_query(LServer, [<<"delete from privacy_list;">>]). - -get_privacy_list_names(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select name from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_privacy_list_names_t(Username) -> - mongoose_rdbms:sql_query_t( - [<<"select name from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -get_privacy_list_id(LServer, Username, SName) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select id from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and name=">>, mongoose_rdbms:use_escaped_string(SName)]). - -get_privacy_list_id_t(Username, SName) -> - mongoose_rdbms:sql_query_t( - [<<"select id from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and name=">>, mongoose_rdbms:use_escaped_string(SName)]). - -get_privacy_list_data(LServer, Username, SName) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select t, value, action, ord, match_all, match_iq, " - "match_message, match_presence_in, match_presence_out " - "from privacy_list_data " - "where id = (select id from privacy_list where " - "username=">>, mongoose_rdbms:use_escaped_string(Username), - <<" and name=">>, mongoose_rdbms:use_escaped_string(SName), <<") " - "order by ord;">>]). - -get_privacy_list_data_by_id(LServer, ID) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select t, value, action, ord, match_all, match_iq, " - "match_message, match_presence_in, match_presence_out " - "from privacy_list_data " - "where id=">>, mongoose_rdbms:use_escaped_integer(ID), <<" order by ord;">>]). - -set_default_privacy_list(Username, SName) -> - update_t(<<"privacy_default_list">>, [<<"username">>, <<"name">>], - [Username, SName], - [<<"username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -unset_default_privacy_list(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from privacy_default_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -remove_privacy_list(Username, SName) -> - mongoose_rdbms:sql_query_t( - [<<"delete from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username), - " and name=", mongoose_rdbms:use_escaped_string(SName)]). - -add_privacy_list(Username, SName) -> - mongoose_rdbms:sql_query_t( - [<<"insert into privacy_list(username, name) " - "values (">>, mongoose_rdbms:use_escaped_string(Username), - ", ", mongoose_rdbms:use_escaped_string(SName), ");"]). - --spec set_privacy_list(mongoose_rdbms:escaped_integer(), - list(list(mongoose_rdbms:escaped_value()))) -> ok. -set_privacy_list(ID, RItems) -> - mongoose_rdbms:sql_query_t( - [<<"delete from privacy_list_data " - "where id=">>, mongoose_rdbms:use_escaped_integer(ID), ";"]), - lists:foreach(fun(Items) -> - mongoose_rdbms:sql_query_t( - [<<"insert into privacy_list_data(" - "id, t, value, action, ord, match_all, match_iq, " - "match_message, match_presence_in, " - "match_presence_out " - ") " - "values (">>, mongoose_rdbms:use_escaped_integer(ID), ", ", - join_escaped(Items), ");"]) - end, RItems). - -del_privacy_lists(LServer, _Server, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from privacy_list_data where id in " - "( select id from privacy_list as pl where pl.username=">>, - mongoose_rdbms:use_escaped_string(Username), <<";">>]), - mongoose_rdbms:sql_query( - LServer, - [<<"delete from privacy_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]), - mongoose_rdbms:sql_query( - LServer, - [<<"delete from privacy_default_list " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - %% Count number of records in a table given a where clause count_records_where(LServer, Table, WhereClause) -> mongoose_rdbms:sql_query( From 17cc2611e9ab6a9c555324d1e8a23693c7965cc3 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 10:31:18 +0100 Subject: [PATCH 67/92] Make select_room_id/select_room_id_and_version prepared --- src/muc_light/mod_muc_light_db_rdbms.erl | 124 ++++++++++++------- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 14 +-- 2 files changed, 80 insertions(+), 58 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index 71d1050529..b81c6838f7 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -72,13 +72,44 @@ %% ------------------------ Backend start/stop ------------------------ -spec start(Host :: jid:server(), MUCHost :: jid:server()) -> ok. -start(_Host, _MUCHost) -> +start(Host, _MUCHost) -> + prepare_queries(Host), ok. -spec stop(Host :: jid:server(), MUCHost :: jid:server()) -> ok. stop(_Host, _MUCHost) -> ok. +%% ------------------------ SQL ------------------------------------------- + +prepare_queries(_Host) -> + mongoose_rdbms:prepare(muc_light_config_delete_all, muc_light_config, [], + <<"DELETE FROM muc_light_config">>), + mongoose_rdbms:prepare(muc_light_occupants_delete_all, muc_light_occupants, [], + <<"DELETE FROM muc_light_occupants">>), + mongoose_rdbms:prepare(muc_light_rooms_delete_all, muc_light_rooms, [], + <<"DELETE FROM muc_light_rooms">>), + mongoose_rdbms:prepare(muc_light_blocking_delete_all, muc_light_blocking, [], + <<"DELETE FROM muc_light_blocking">>), + + mongoose_rdbms:prepare(muc_light_select_room_id, muc_light_rooms, + [luser, lserver], + <<"SELECT id FROM muc_light_rooms " + "WHERE luser = ? AND lserver = ?">>), + mongoose_rdbms:prepare(muc_light_select_room_id_and_version, muc_light_rooms, + [luser, lserver], + <<"SELECT id, version FROM muc_light_rooms " + "WHERE luser = ? AND lserver = ?">>), + ok. + +select_room_id(MainHost, RoomU, RoomS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_room_id, [RoomU, RoomS]). + +select_room_id_and_version(MainHost, RoomU, RoomS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_room_id_and_version, [RoomU, RoomS]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), @@ -86,23 +117,21 @@ stop(_Host, _MUCHost) -> {ok, FinalRoomUS :: jid:simple_bare_jid()} | {error, exists}. create_room(RoomUS, Config, AffUsers, Version) -> MainHost = main_host(RoomUS), - {atomic, Res} - = mongoose_rdbms:sql_transaction( - MainHost, fun() -> create_room_transaction(RoomUS, Config, AffUsers, Version) end), + F = fun() -> create_room_transaction(MainHost, RoomUS, Config, AffUsers, Version) end, + {atomic, Res} = mongoose_rdbms:sql_transaction(MainHost, F), Res. -spec destroy_room(RoomUS :: jid:simple_bare_jid()) -> ok | {error, not_exists | not_empty}. destroy_room(RoomUS) -> MainHost = main_host(RoomUS), - {atomic, Res} - = mongoose_rdbms:sql_transaction(MainHost, fun() -> destroy_room_transaction(RoomUS) end), + F = fun() -> destroy_room_transaction(MainHost, RoomUS) end, + {atomic, Res} = mongoose_rdbms:sql_transaction(MainHost, F), Res. -spec room_exists(RoomUS :: jid:simple_bare_jid()) -> boolean(). room_exists({RoomU, RoomS} = RoomUS) -> MainHost = main_host(RoomUS), - {selected, Res} = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_room_id(RoomU, RoomS)), + {selected, Res} = select_room_id(MainHost, RoomU, RoomS), Res /= []. -spec get_user_rooms(UserUS :: jid:simple_bare_jid(), @@ -134,8 +163,8 @@ get_user_rooms_count({LUser, LServer}, MUCServer) -> -spec remove_user(UserUS :: jid:simple_bare_jid(), Version :: binary()) -> mod_muc_light_db:remove_user_return() | {error, term()}. remove_user({_, UserS} = UserUS, Version) -> - {atomic, Res} - = mongoose_rdbms:sql_transaction(UserS, fun() -> remove_user_transaction(UserUS, Version) end), + F = fun() -> remove_user_transaction(UserS, UserUS, Version) end, + {atomic, Res} = mongoose_rdbms:sql_transaction(UserS, F), Res. %% ------------------------ Configuration manipulation ------------------------ @@ -261,10 +290,9 @@ get_aff_users({RoomU, RoomS} = RoomUS) -> mod_muc_light_db:modify_aff_users_return(). modify_aff_users(RoomUS, AffUsersChanges, ExternalCheck, Version) -> MainHost = main_host(RoomUS), - {atomic, Res} - = mongoose_rdbms:sql_transaction( - MainHost, fun() -> modify_aff_users_transaction( - RoomUS, AffUsersChanges, ExternalCheck, Version) end), + F = fun() -> modify_aff_users_transaction(MainHost, RoomUS, AffUsersChanges, + ExternalCheck, Version) end, + {atomic, Res} = mongoose_rdbms:sql_transaction(MainHost, F), Res. %% ------------------------ Misc ------------------------ @@ -274,8 +302,7 @@ modify_aff_users(RoomUS, AffUsersChanges, ExternalCheck, Version) -> | {error, not_exists}. get_info({RoomU, RoomS} = RoomUS) -> MainHost = main_host(RoomUS), - case mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_room_id_and_version(RoomU, RoomS)) of + case select_room_id_and_version(MainHost, RoomU, RoomS) of {selected, [{RoomID, Version}]} -> {selected, AffUsersDB} = mongoose_rdbms:sql_query( MainHost, mod_muc_light_db_rdbms_sql:select_affs(RoomID)), @@ -315,15 +342,17 @@ aff_db2atom(Bin) -> aff_db2atom(mongoose_rdbms:result_to_integer(Bin)). %% API for tests %%==================================================================== +force_clear_statements() -> + [muc_light_config_delete_all, + muc_light_occupants_delete_all, + muc_light_rooms_delete_all, + muc_light_blocking_delete_all]. + -spec force_clear() -> ok. force_clear() -> - lists:foreach( - fun(Host) -> - mongoose_rdbms:sql_query(Host, ["DELETE FROM muc_light_config"]), - mongoose_rdbms:sql_query(Host, ["DELETE FROM muc_light_occupants"]), - mongoose_rdbms:sql_query(Host, ["DELETE FROM muc_light_rooms"]), - mongoose_rdbms:sql_query(Host, ["DELETE FROM muc_light_blocking"]) - end, ?MYHOSTS). + [mongoose_rdbms:execute_successfully(Host, Statement, []) + || Host <- ?MYHOSTS, Statement <- force_clear_statements()], + ok. %%==================================================================== %% Internal functions @@ -332,12 +361,13 @@ force_clear() -> %% ------------------------ General room management ------------------------ %% Expects config to have unique fields! --spec create_room_transaction(RoomUS :: jid:simple_bare_jid(), +-spec create_room_transaction(MainHost :: jid:lserver(), + RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), AffUsers :: aff_users(), Version :: binary()) -> {ok, FinalRoomUS :: jid:simple_bare_jid()} | {error, exists}. -create_room_transaction({NodeCandidate, RoomS}, Config, AffUsers, Version) -> +create_room_transaction(MainHost, {NodeCandidate, RoomS}, Config, AffUsers, Version) -> RoomU = case NodeCandidate of <<>> -> mongoose_bin:gen_from_timestamp(); _ -> NodeCandidate @@ -349,18 +379,16 @@ create_room_transaction({NodeCandidate, RoomS}, Config, AffUsers, Version) -> mongoose_rdbms:sql_query_t("ROLLBACK;"), mongoose_rdbms:sql_query_t(rdbms_queries:begin_trans()), - case {mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:select_room_id(RoomU, RoomS)), NodeCandidate} of + case {select_room_id(MainHost, RoomU, RoomS), NodeCandidate} of {{selected, []}, _} -> throw({aborted, Reason}); {{selected, [_]}, <<>>} -> - create_room_transaction({<<>>, RoomS}, Config, AffUsers, Version); + create_room_transaction(MainHost, {<<>>, RoomS}, Config, AffUsers, Version); {{selected, [_]}, _} -> {error, exists} end; {updated, _} -> - {selected, [{RoomID} | Rest] = AllIds} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:select_room_id(RoomU, RoomS)), + {selected, [{RoomID} | Rest] = AllIds} = select_room_id(MainHost, RoomU, RoomS), case Rest of [] -> ok; @@ -385,9 +413,11 @@ create_room_transaction({NodeCandidate, RoomS}, Config, AffUsers, Version) -> {ok, {RoomU, RoomS}} end. --spec destroy_room_transaction(RoomUS :: jid:simple_bare_jid()) -> ok | {error, not_exists}. -destroy_room_transaction({RoomU, RoomS}) -> - case mongoose_rdbms:sql_query_t(mod_muc_light_db_rdbms_sql:select_room_id(RoomU, RoomS)) of +-spec destroy_room_transaction(MainHost :: jid:lserver(), + RoomUS :: jid:simple_bare_jid()) -> + ok | {error, not_exists}. +destroy_room_transaction(MainHost, {RoomU, RoomS}) -> + case select_room_id(MainHost, RoomU, RoomS) of {selected, [{RoomID}]} -> {updated, _} = mongoose_rdbms:sql_query_t( mod_muc_light_db_rdbms_sql:delete_affs(RoomID)), @@ -400,15 +430,17 @@ destroy_room_transaction({RoomU, RoomS}) -> {error, not_exists} end. --spec remove_user_transaction(UserUS :: jid:simple_bare_jid(), Version :: binary()) -> +-spec remove_user_transaction(MainHost :: jid:lserver(), + UserUS :: jid:simple_bare_jid(), + Version :: binary()) -> mod_muc_light_db:remove_user_return(). -remove_user_transaction({UserU, UserS} = UserUS, Version) -> +remove_user_transaction(MainHost, {UserU, UserS} = UserUS, Version) -> Rooms = get_user_rooms(UserUS, undefined), {updated, _} = mongoose_rdbms:sql_query_t( mod_muc_light_db_rdbms_sql:delete_blocking(UserU, UserS)), lists:map( fun(RoomUS) -> - {RoomUS, modify_aff_users_transaction( + {RoomUS, modify_aff_users_transaction(MainHost, RoomUS, [{UserUS, none}], fun(_, _) -> ok end, Version)} end, Rooms). @@ -420,8 +452,7 @@ remove_user_transaction({UserU, UserS} = UserUS, Version) -> {ok, PrevVersion :: binary()} | {error, not_exists}. set_config_transaction({RoomU, RoomS} = RoomUS, ConfigChanges, Version) -> MainHost = main_host(RoomUS), - case mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:select_room_id_and_version(RoomU, RoomS)) of + case select_room_id_and_version(MainHost, RoomU, RoomS) of {selected, [{RoomID, PrevVersion}]} -> {updated, _} = mongoose_rdbms:sql_query_t( mod_muc_light_db_rdbms_sql:update_room_version(RoomU, RoomS, Version)), @@ -441,29 +472,32 @@ set_config_transaction({RoomU, RoomS} = RoomUS, ConfigChanges, Version) -> %% ------------------------ Affiliations manipulation ------------------------ --spec modify_aff_users_transaction(RoomUS :: jid:simple_bare_jid(), +-spec modify_aff_users_transaction(MainHost :: jid:lserver(), + RoomUS :: jid:simple_bare_jid(), AffUsersChanges :: aff_users(), CheckFun :: external_check_fun(), Version :: binary()) -> mod_muc_light_db:modify_aff_users_return(). -modify_aff_users_transaction({RoomU, RoomS} = RoomUS, AffUsersChanges, CheckFun, Version) -> - case mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:select_room_id_and_version(RoomU, RoomS)) of +modify_aff_users_transaction(MainHost, {RoomU, RoomS} = RoomUS, + AffUsersChanges, CheckFun, Version) -> + case select_room_id_and_version(MainHost, RoomU, RoomS) of {selected, [{RoomID, PrevVersion}]} -> - modify_aff_users_transaction( + modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, CheckFun, PrevVersion, Version); {selected, []} -> {error, not_exists} end. --spec modify_aff_users_transaction(RoomUS :: jid:simple_bare_jid(), +-spec modify_aff_users_transaction(MainHost :: jid:lserver(), + RoomUS :: jid:simple_bare_jid(), RoomID :: binary(), AffUsersChanges :: aff_users(), CheckFun :: external_check_fun(), PrevVersion :: binary(), Version :: binary()) -> mod_muc_light_db:modify_aff_users_return(). -modify_aff_users_transaction(RoomUS, RoomID, AffUsersChanges, CheckFun, PrevVersion, Version) -> +modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, + CheckFun, PrevVersion, Version) -> {selected, AffUsersDB} = mongoose_rdbms:sql_query_t(mod_muc_light_db_rdbms_sql:select_affs(RoomID)), AffUsers = lists:sort( diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl index 7c8cb447a3..5fc7a7ece6 100644 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ b/src/muc_light/mod_muc_light_db_rdbms_sql.erl @@ -25,8 +25,7 @@ -include("mod_muc_light.hrl"). --export([select_room_id/2, select_room_id_and_version/2, - select_user_rooms/2, select_user_rooms_count/2, +-export([select_user_rooms/2, select_user_rooms_count/2, insert_room/3, update_room_version/3, delete_room/2]). -export([select_affs/2, select_affs/1, insert_aff/4, update_aff/4, delete_affs/1, delete_aff/3]). -export([select_config/1, select_config/2, select_config/3, insert_config/3, update_config/3, @@ -40,17 +39,6 @@ %% General room queries %%==================================================================== --spec select_room_id(RoomU :: jid:luser(), RoomS :: jid:lserver()) -> iolist(). -select_room_id(RoomU, RoomS) -> - ["SELECT id FROM muc_light_rooms WHERE luser = ", ?ESC(RoomU), - " AND lserver = ", ?ESC(RoomS)]. - --spec select_room_id_and_version( - RoomU :: jid:luser(), RoomS :: jid:lserver()) -> iolist(). -select_room_id_and_version(RoomU, RoomS) -> - ["SELECT id, version FROM muc_light_rooms WHERE luser = ", ?ESC(RoomU), - " AND lserver = ", ?ESC(RoomS)]. - -spec select_user_rooms(LUser :: jid:luser(), LServer :: jid:lserver()) -> iolist(). select_user_rooms(LUser, LServer) -> select_user_rooms(LUser, LServer, "r.luser, r.lserver"). From 23a1d9d64077de6993182848454929b0a6493924 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 11:08:54 +0100 Subject: [PATCH 68/92] Split create_room function into two Prepare query for muc_light_insert_room function --- src/muc_light/mod_muc_light_db_rdbms.erl | 145 ++++++++++++------- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 25 +--- 2 files changed, 91 insertions(+), 79 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index b81c6838f7..74d29ae63e 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -92,6 +92,7 @@ prepare_queries(_Host) -> mongoose_rdbms:prepare(muc_light_blocking_delete_all, muc_light_blocking, [], <<"DELETE FROM muc_light_blocking">>), + %% Returns maximum 1 record mongoose_rdbms:prepare(muc_light_select_room_id, muc_light_rooms, [luser, lserver], <<"SELECT id FROM muc_light_rooms " @@ -100,6 +101,25 @@ prepare_queries(_Host) -> [luser, lserver], <<"SELECT id, version FROM muc_light_rooms " "WHERE luser = ? AND lserver = ?">>), + mongoose_rdbms:prepare(muc_light_insert_room, muc_light_rooms, + [luser, lserver, version], + <<"INSERT INTO muc_light_rooms (luser, lserver, version)" + " VALUES (?, ?, ?)">>), + + %% This query uses multiple table + mongoose_rdbms:prepare(muc_light_select_user_rooms, muc_light_occupants, + [luser, lserver], + <<"SELECT r.luser, r.lserver " + " FROM muc_light_occupants AS o " + " INNER JOIN muc_light_rooms AS r ON o.room_id = r.id" + " WHERE o.luser = ? AND o.lserver = ?">>), + mongoose_rdbms:prepare(muc_light_select_user_rooms_count, muc_light_occupants, + [luser, lserver], + <<"SELECT count(*) " + " FROM muc_light_occupants AS o " + " INNER JOIN muc_light_rooms AS r ON o.room_id = r.id" + " WHERE o.luser = ? AND o.lserver = ?">>), + ok. select_room_id(MainHost, RoomU, RoomS) -> @@ -110,16 +130,64 @@ select_room_id_and_version(MainHost, RoomU, RoomS) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_select_room_id_and_version, [RoomU, RoomS]). +select_user_rooms(MainHost, LUser, LServer) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_user_rooms, [LUser, LServer]). + +select_user_rooms_count(MainHost, LUser, LServer) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_user_rooms_count, [LUser, LServer]). + +insert_room(MainHost, RoomU, RoomS, Version) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_insert_room, [RoomU, RoomS, Version]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), AffUsers :: aff_users(), Version :: binary()) -> {ok, FinalRoomUS :: jid:simple_bare_jid()} | {error, exists}. +create_room({<<>>, RoomS} = RoomUS, Config, AffUsers, Version) -> + MainHost = main_host(RoomUS), + create_room_with_random_name(MainHost, RoomS, Config, AffUsers, Version, 10); create_room(RoomUS, Config, AffUsers, Version) -> MainHost = main_host(RoomUS), + create_room_with_specified_name(MainHost, RoomUS, Config, AffUsers, Version). + +create_room_with_random_name(_MainHost, RoomS, _Config, _AffUsers, _Version, 0) -> + ?LOG_ERROR(#{what => muc_create_room_with_random_name_failed, + sub_host => RoomS}), + error(create_room_with_random_name_failed); +create_room_with_random_name(MainHost, RoomS, Config, AffUsers, Version, Retries) + when Retries > 0 -> + RoomU = mongoose_bin:gen_from_timestamp(), + RoomUS = {RoomU, RoomS}, F = fun() -> create_room_transaction(MainHost, RoomUS, Config, AffUsers, Version) end, - {atomic, Res} = mongoose_rdbms:sql_transaction(MainHost, F), - Res. + case mongoose_rdbms:sql_transaction(MainHost, F) of + {atomic, ok} -> + {ok, RoomUS}; + Other -> + ?LOG_ERROR(#{what => muc_create_room_with_random_name_retry, + candidate_room => RoomU, sub_host => RoomS, reason => Other}), + create_room_with_random_name(MainHost, RoomS, Config, AffUsers, Version, Retries-1) + end. + +create_room_with_specified_name(MainHost, RoomUS, Config, AffUsers, Version) -> + F = fun() -> create_room_transaction(MainHost, RoomUS, Config, AffUsers, Version) end, + case mongoose_rdbms:sql_transaction(MainHost, F) of + {atomic, ok} -> + {ok, RoomUS}; + Other -> + case room_exists(RoomUS) of + true -> + {error, exists}; + false -> %% Something unknown + {RoomU, RoomS} = RoomUS, + ?LOG_ERROR(#{what => muc_create_room_with_specified_name_failed, + room => RoomU, sub_host => RoomS, reason => Other}), + error(create_room_with_specified_name_failed) + end + end. -spec destroy_room(RoomUS :: jid:simple_bare_jid()) -> ok | {error, not_exists | not_empty}. destroy_room(RoomUS) -> @@ -138,16 +206,14 @@ room_exists({RoomU, RoomS} = RoomUS) -> MUCServer :: jid:lserver() | undefined) -> [RoomUS :: jid:simple_bare_jid()]. get_user_rooms({LUser, LServer}, undefined) -> - SQL = mod_muc_light_db_rdbms_sql:select_user_rooms(LUser, LServer), lists:usort(lists:flatmap( fun(Host) -> - {selected, Rooms} = mongoose_rdbms:sql_query(Host, SQL), + {selected, Rooms} = select_user_rooms(Host, LUser, LServer), Rooms end, ?MYHOSTS)); get_user_rooms({LUser, LServer}, MUCServer) -> MainHost = main_host(MUCServer), - {selected, Rooms} = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_user_rooms(LUser, LServer)), + {selected, Rooms} = select_user_rooms(MainHost, LUser, LServer), Rooms. -spec get_user_rooms_count(UserUS :: jid:simple_bare_jid(), @@ -155,9 +221,7 @@ get_user_rooms({LUser, LServer}, MUCServer) -> non_neg_integer(). get_user_rooms_count({LUser, LServer}, MUCServer) -> MainHost = main_host(MUCServer), - {selected, [{Cnt}]} - = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_user_rooms_count(LUser, LServer)), + {selected, [{Cnt}]} = select_user_rooms_count(MainHost, LUser, LServer), mongoose_rdbms:result_to_integer(Cnt). -spec remove_user(UserUS :: jid:simple_bare_jid(), Version :: binary()) -> @@ -366,52 +430,23 @@ force_clear() -> Config :: mod_muc_light_room_config:kv(), AffUsers :: aff_users(), Version :: binary()) -> - {ok, FinalRoomUS :: jid:simple_bare_jid()} | {error, exists}. -create_room_transaction(MainHost, {NodeCandidate, RoomS}, Config, AffUsers, Version) -> - RoomU = case NodeCandidate of - <<>> -> mongoose_bin:gen_from_timestamp(); - _ -> NodeCandidate - end, - case catch mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:insert_room(RoomU, RoomS, Version)) of - {aborted, Reason} -> - %% At this point the transaction is broken because of failed INSERT query - mongoose_rdbms:sql_query_t("ROLLBACK;"), - mongoose_rdbms:sql_query_t(rdbms_queries:begin_trans()), - - case {select_room_id(MainHost, RoomU, RoomS), NodeCandidate} of - {{selected, []}, _} -> - throw({aborted, Reason}); - {{selected, [_]}, <<>>} -> - create_room_transaction(MainHost, {<<>>, RoomS}, Config, AffUsers, Version); - {{selected, [_]}, _} -> - {error, exists} - end; - {updated, _} -> - {selected, [{RoomID} | Rest] = AllIds} = select_room_id(MainHost, RoomU, RoomS), - case Rest of - [] -> - ok; - _ -> - Details = <<"Many IDs returned for PK select query, most probably MSSQL deadlock">>, - ?LOG_ERROR(#{what => muc_many_ids_for_pk_select, text => Details, - room => RoomU, sub_host => RoomS, all_room_ids => AllIds}), - throw({aborted, Details}) - end, - lists:foreach( - fun({{UserU, UserS}, Aff}) -> - Query = mod_muc_light_db_rdbms_sql:insert_aff(RoomID, UserU, UserS, Aff), - {updated, _} = mongoose_rdbms:sql_query_t(Query) - end, AffUsers), - ConfigFields = mod_muc_light_room_config:to_binary_kv( - Config, mod_muc_light:config_schema(RoomS)), - lists:foreach( - fun({Key, Val}) -> - Query = mod_muc_light_db_rdbms_sql:insert_config(RoomID, Key, Val), - {updated, _} = mongoose_rdbms:sql_query_t(Query) - end, ConfigFields), - {ok, {RoomU, RoomS}} - end. + ok | {error, exists}. +create_room_transaction(MainHost, {RoomU, RoomS}, Config, AffUsers, Version) -> + insert_room(MainHost, RoomU, RoomS, Version), + {selected, [{RoomID}]} = select_room_id(MainHost, RoomU, RoomS), + lists:foreach( + fun({{UserU, UserS}, Aff}) -> + Query = mod_muc_light_db_rdbms_sql:insert_aff(RoomID, UserU, UserS, Aff), + {updated, _} = mongoose_rdbms:sql_query_t(Query) + end, AffUsers), + ConfigFields = mod_muc_light_room_config:to_binary_kv( + Config, mod_muc_light:config_schema(RoomS)), + lists:foreach( + fun({Key, Val}) -> + Query = mod_muc_light_db_rdbms_sql:insert_config(RoomID, Key, Val), + {updated, _} = mongoose_rdbms:sql_query_t(Query) + end, ConfigFields), + ok. -spec destroy_room_transaction(MainHost :: jid:lserver(), RoomUS :: jid:simple_bare_jid()) -> diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl index 5fc7a7ece6..7b8f1e15ca 100644 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ b/src/muc_light/mod_muc_light_db_rdbms_sql.erl @@ -25,8 +25,7 @@ -include("mod_muc_light.hrl"). --export([select_user_rooms/2, select_user_rooms_count/2, - insert_room/3, update_room_version/3, delete_room/2]). +-export([update_room_version/3, delete_room/2]). -export([select_affs/2, select_affs/1, insert_aff/4, update_aff/4, delete_affs/1, delete_aff/3]). -export([select_config/1, select_config/2, select_config/3, insert_config/3, update_config/3, delete_config/1]). @@ -39,28 +38,6 @@ %% General room queries %%==================================================================== --spec select_user_rooms(LUser :: jid:luser(), LServer :: jid:lserver()) -> iolist(). -select_user_rooms(LUser, LServer) -> - select_user_rooms(LUser, LServer, "r.luser, r.lserver"). - --spec select_user_rooms_count(LUser :: jid:luser(), LServer :: jid:lserver()) -> iolist(). -select_user_rooms_count(LUser, LServer) -> - select_user_rooms(LUser, LServer, "COUNT(*)"). - --spec select_user_rooms(LUser :: jid:luser(), - LServer :: jid:lserver(), - ReturnStatement :: iodata()) -> iolist(). -select_user_rooms(LUser, LServer, ReturnStatement) -> - ["SELECT ", ReturnStatement, - " FROM muc_light_occupants AS o INNER JOIN muc_light_rooms AS r ON o.room_id = r.id" - " WHERE o.luser = ", ?ESC(LUser), " AND o.lserver = ", ?ESC(LServer)]. - --spec insert_room( - RoomU :: jid:luser(), RoomS :: jid:lserver(), Version :: binary()) -> iolist(). -insert_room(RoomU, RoomS, Version) -> - ["INSERT INTO muc_light_rooms (luser, lserver, version)" - " VALUES (", ?ESC(RoomU), ", ", ?ESC(RoomS), ", ", ?ESC(Version), ")"]. - -spec update_room_version( RoomU :: jid:luser(), RoomS :: jid:lserver(), Version :: binary()) -> iolist(). update_room_version(RoomU, RoomS, Version) -> From 2ebd1c316c0156c059cace7dc66f1cd7ceab6416 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 12:27:26 +0100 Subject: [PATCH 69/92] Use prepared query for update_room_version and delete_room functions --- src/muc_light/mod_muc_light_db_rdbms.erl | 31 +++++++++++++------- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 16 ---------- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index 74d29ae63e..c0f690e39d 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -105,6 +105,14 @@ prepare_queries(_Host) -> [luser, lserver, version], <<"INSERT INTO muc_light_rooms (luser, lserver, version)" " VALUES (?, ?, ?)">>), + mongoose_rdbms:prepare(muc_light_update_room_version, muc_light_rooms, + [luser, lserver, version], + <<"UPDATE muc_light_rooms SET version = ? " + " WHERE luser = ? AND lserver = ?">>), + mongoose_rdbms:prepare(muc_light_delete_room, muc_light_rooms, + [luser, lserver], + <<"DELETE FROM muc_light_rooms" + " WHERE luser = ? AND lserver = ?">>), %% This query uses multiple table mongoose_rdbms:prepare(muc_light_select_user_rooms, muc_light_occupants, @@ -142,6 +150,14 @@ insert_room(MainHost, RoomU, RoomS, Version) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_insert_room, [RoomU, RoomS, Version]). +update_room_version(MainHost, RoomU, RoomS, Version) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_update_room_version, [Version, RoomU, RoomS]). + +delete_room(MainHost, RoomU, RoomS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_room, [RoomU, RoomS]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), @@ -458,8 +474,7 @@ destroy_room_transaction(MainHost, {RoomU, RoomS}) -> mod_muc_light_db_rdbms_sql:delete_affs(RoomID)), {updated, _} = mongoose_rdbms:sql_query_t( mod_muc_light_db_rdbms_sql:delete_config(RoomID)), - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:delete_room(RoomU, RoomS)), + {updated, _} = delete_room(MainHost, RoomU, RoomS), ok; {selected, []} -> {error, not_exists} @@ -489,8 +504,7 @@ set_config_transaction({RoomU, RoomS} = RoomUS, ConfigChanges, Version) -> MainHost = main_host(RoomUS), case select_room_id_and_version(MainHost, RoomU, RoomS) of {selected, [{RoomID, PrevVersion}]} -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:update_room_version(RoomU, RoomS, Version)), + {updated, _} = update_room_version(MainHost, RoomU, RoomS, Version), lists:foreach( fun({Key, Val}) -> {updated, _} @@ -542,7 +556,8 @@ modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, case CheckFun(RoomUS, NewAffUsers) of ok -> apply_aff_users_transaction(RoomID, AffUsersChanged, JoiningUsers), - update_room_version_transaction(RoomUS, Version), + {RoomU, RoomS} = RoomUS, + {updated, _} = update_room_version(MainHost, RoomU, RoomS, Version), {ok, AffUsers, NewAffUsers, AffUsersChanged, PrevVersion}; Error -> Error @@ -572,12 +587,6 @@ apply_aff_users_transaction(RoomID, AffUsersChanged, JoiningUsers) -> end end, AffUsersChanged). --spec update_room_version_transaction(RoomUS :: jid:simple_bare_jid(), Version :: binary()) -> - {updated, integer()}. -update_room_version_transaction({RoomU, RoomS}, Version) -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:update_room_version(RoomU, RoomS, Version)). - %% ------------------------ Common ------------------------ -spec main_host(JIDOrServer :: jid:simple_bare_jid() | binary()) -> jid:lserver(). diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl index 7b8f1e15ca..f56926bf15 100644 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ b/src/muc_light/mod_muc_light_db_rdbms_sql.erl @@ -25,7 +25,6 @@ -include("mod_muc_light.hrl"). --export([update_room_version/3, delete_room/2]). -export([select_affs/2, select_affs/1, insert_aff/4, update_aff/4, delete_affs/1, delete_aff/3]). -export([select_config/1, select_config/2, select_config/3, insert_config/3, update_config/3, delete_config/1]). @@ -34,21 +33,6 @@ -define(ESC(T), mongoose_rdbms:use_escaped_string(mongoose_rdbms:escape_string(T))). -%%==================================================================== -%% General room queries -%%==================================================================== - --spec update_room_version( - RoomU :: jid:luser(), RoomS :: jid:lserver(), Version :: binary()) -> iolist(). -update_room_version(RoomU, RoomS, Version) -> - ["UPDATE muc_light_rooms SET version = ", ?ESC(Version), - " WHERE luser = ", ?ESC(RoomU), " AND lserver = ", ?ESC(RoomS)]. - --spec delete_room(RoomU :: jid:luser(), RoomS :: jid:lserver()) -> iolist(). -delete_room(RoomU, RoomS) -> - ["DELETE FROM muc_light_rooms" - " WHERE luser = ", ?ESC(RoomU), " AND lserver = ", ?ESC(RoomS)]. - %%==================================================================== %% Affiliations %%==================================================================== From 2a991dcede489bdef4a1f164b9760caf21591b7f Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 12:51:22 +0100 Subject: [PATCH 70/92] Use prepared function for select_affs/2 in MUC-light --- src/muc_light/mod_muc_light_db_rdbms.erl | 30 ++++++++++++++++---- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 8 +----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index c0f690e39d..467bb05fc6 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -82,7 +82,7 @@ stop(_Host, _MUCHost) -> %% ------------------------ SQL ------------------------------------------- -prepare_queries(_Host) -> +prepare_queries(Host) -> mongoose_rdbms:prepare(muc_light_config_delete_all, muc_light_config, [], <<"DELETE FROM muc_light_config">>), mongoose_rdbms:prepare(muc_light_occupants_delete_all, muc_light_occupants, [], @@ -91,7 +91,11 @@ prepare_queries(_Host) -> <<"DELETE FROM muc_light_rooms">>), mongoose_rdbms:prepare(muc_light_blocking_delete_all, muc_light_blocking, [], <<"DELETE FROM muc_light_blocking">>), + prepare_room_queries(Host), + prepare_affiliation_queries(Host), + ok. +prepare_room_queries(_Host) -> %% Returns maximum 1 record mongoose_rdbms:prepare(muc_light_select_room_id, muc_light_rooms, [luser, lserver], @@ -113,7 +117,6 @@ prepare_queries(_Host) -> [luser, lserver], <<"DELETE FROM muc_light_rooms" " WHERE luser = ? AND lserver = ?">>), - %% This query uses multiple table mongoose_rdbms:prepare(muc_light_select_user_rooms, muc_light_occupants, [luser, lserver], @@ -127,9 +130,20 @@ prepare_queries(_Host) -> " FROM muc_light_occupants AS o " " INNER JOIN muc_light_rooms AS r ON o.room_id = r.id" " WHERE o.luser = ? AND o.lserver = ?">>), - ok. +prepare_affiliation_queries(Host) -> + %% This query uses multiple table + mongoose_rdbms:prepare(muc_light_select_affs_by_us, muc_light_rooms, + [luser, lserver], + <<"SELECT version, o.luser, o.lserver, aff" + " FROM muc_light_rooms AS r " + " LEFT OUTER JOIN muc_light_occupants AS o ON r.id = o.room_id" + " WHERE r.luser = ? AND r.lserver = ?">>), + ok. + +%% ------------------------ Room SQL functions ------------------------ + select_room_id(MainHost, RoomU, RoomS) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_select_room_id, [RoomU, RoomS]). @@ -158,6 +172,12 @@ delete_room(MainHost, RoomU, RoomS) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_delete_room, [RoomU, RoomS]). +%% ------------------------ Affiliation SQL functions ------------------------ + +select_affs_by_us(MainHost, RoomU, RoomS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_affs_by_us, [RoomU, RoomS]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), @@ -197,7 +217,7 @@ create_room_with_specified_name(MainHost, RoomUS, Config, AffUsers, Version) -> case room_exists(RoomUS) of true -> {error, exists}; - false -> %% Something unknown + false -> %% Some unknown error condition {RoomU, RoomS} = RoomUS, ?LOG_ERROR(#{what => muc_create_room_with_specified_name_failed, room => RoomU, sub_host => RoomS, reason => Other}), @@ -353,7 +373,7 @@ set_blocking({LUser, LServer} = UserUS, MUCServer, [{What, allow, Who} | RBlocki {ok, aff_users(), Version :: binary()} | {error, not_exists}. get_aff_users({RoomU, RoomS} = RoomUS) -> MainHost = main_host(RoomUS), - case mongoose_rdbms:sql_query(MainHost, mod_muc_light_db_rdbms_sql:select_affs(RoomU, RoomS)) of + case select_affs_by_us(MainHost, RoomU, RoomS) of {selected, []} -> {error, not_exists}; {selected, [{Version, null, null, null}]} -> diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl index f56926bf15..6bd716b32d 100644 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ b/src/muc_light/mod_muc_light_db_rdbms_sql.erl @@ -25,7 +25,7 @@ -include("mod_muc_light.hrl"). --export([select_affs/2, select_affs/1, insert_aff/4, update_aff/4, delete_affs/1, delete_aff/3]). +-export([select_affs/1, insert_aff/4, update_aff/4, delete_affs/1, delete_aff/3]). -export([select_config/1, select_config/2, select_config/3, insert_config/3, update_config/3, delete_config/1]). -export([select_blocking/2, select_blocking_cnt/3, insert_blocking/4, @@ -37,12 +37,6 @@ %% Affiliations %%==================================================================== --spec select_affs(RoomU :: jid:luser(), RoomS :: jid:lserver()) -> iolist(). -select_affs(RoomU, RoomS) -> - ["SELECT version, o.luser, o.lserver, aff" - " FROM muc_light_rooms AS r LEFT OUTER JOIN muc_light_occupants AS o ON r.id = o.room_id" - " WHERE r.luser = ", ?ESC(RoomU), " AND r.lserver = ", ?ESC(RoomS)]. - -spec select_affs(RoomID :: integer() | binary()) -> iolist(). select_affs(RoomID) -> ["SELECT luser, lserver, aff FROM muc_light_occupants WHERE room_id = ", bin(RoomID)]. From feff19a9aee156e044271508285a84b477c0e8e8 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 12:55:05 +0100 Subject: [PATCH 71/92] Use prepared function for select_affs/1 in MUC-light --- src/muc_light/mod_muc_light_db_rdbms.erl | 14 ++++++++++---- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 6 +----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index 467bb05fc6..c89bad3dbe 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -140,6 +140,10 @@ prepare_affiliation_queries(Host) -> " FROM muc_light_rooms AS r " " LEFT OUTER JOIN muc_light_occupants AS o ON r.id = o.room_id" " WHERE r.luser = ? AND r.lserver = ?">>), + mongoose_rdbms:prepare(muc_light_select_affs_by_room_id, muc_light_occupants, + [room_id], + <<"SELECT luser, lserver, aff " + "FROM muc_light_occupants WHERE room_id = ?">>), ok. %% ------------------------ Room SQL functions ------------------------ @@ -178,6 +182,10 @@ select_affs_by_us(MainHost, RoomU, RoomS) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_select_affs_by_us, [RoomU, RoomS]). +select_affs_by_room_id(MainHost, RoomID) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_affs_by_room_id, [RoomID]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), @@ -404,8 +412,7 @@ get_info({RoomU, RoomS} = RoomUS) -> MainHost = main_host(RoomUS), case select_room_id_and_version(MainHost, RoomU, RoomS) of {selected, [{RoomID, Version}]} -> - {selected, AffUsersDB} = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_affs(RoomID)), + {selected, AffUsersDB} = select_affs_by_room_id(MainHost, RoomID), AffUsers = [{{UserU, UserS}, aff_db2atom(Aff)} || {UserU, UserS, Aff} <- AffUsersDB], {selected, ConfigDB} = mongoose_rdbms:sql_query( @@ -567,8 +574,7 @@ modify_aff_users_transaction(MainHost, {RoomU, RoomS} = RoomUS, mod_muc_light_db:modify_aff_users_return(). modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, CheckFun, PrevVersion, Version) -> - {selected, AffUsersDB} - = mongoose_rdbms:sql_query_t(mod_muc_light_db_rdbms_sql:select_affs(RoomID)), + {selected, AffUsersDB} = select_affs_by_room_id(MainHost, RoomID), AffUsers = lists:sort( [{{UserU, UserS}, aff_db2atom(Aff)} || {UserU, UserS, Aff} <- AffUsersDB]), case mod_muc_light_utils:change_aff_users(AffUsers, AffUsersChanges) of diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl index 6bd716b32d..0c6132dd58 100644 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ b/src/muc_light/mod_muc_light_db_rdbms_sql.erl @@ -25,7 +25,7 @@ -include("mod_muc_light.hrl"). --export([select_affs/1, insert_aff/4, update_aff/4, delete_affs/1, delete_aff/3]). +-export([insert_aff/4, update_aff/4, delete_affs/1, delete_aff/3]). -export([select_config/1, select_config/2, select_config/3, insert_config/3, update_config/3, delete_config/1]). -export([select_blocking/2, select_blocking_cnt/3, insert_blocking/4, @@ -37,10 +37,6 @@ %% Affiliations %%==================================================================== --spec select_affs(RoomID :: integer() | binary()) -> iolist(). -select_affs(RoomID) -> - ["SELECT luser, lserver, aff FROM muc_light_occupants WHERE room_id = ", bin(RoomID)]. - -spec insert_aff(RoomID :: integer() | binary(), UserU :: jid:luser(), UserS :: jid:lserver(), Aff :: aff()) -> iolist(). insert_aff(RoomID, UserU, UserS, Aff) -> From 9e063a39cac611fa819cbd5881688947c37b45d2 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 13:08:14 +0100 Subject: [PATCH 72/92] Add decode_affs/1 function --- src/muc_light/mod_muc_light_db_rdbms.erl | 25 +++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index c89bad3dbe..9a2d12fb55 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -134,6 +134,7 @@ prepare_room_queries(_Host) -> prepare_affiliation_queries(Host) -> %% This query uses multiple table + %% Also returns a room version mongoose_rdbms:prepare(muc_light_select_affs_by_us, muc_light_rooms, [luser, lserver], <<"SELECT version, o.luser, o.lserver, aff" @@ -178,10 +179,12 @@ delete_room(MainHost, RoomU, RoomS) -> %% ------------------------ Affiliation SQL functions ------------------------ +%% Returns affiliations with a version select_affs_by_us(MainHost, RoomU, RoomS) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_select_affs_by_us, [RoomU, RoomS]). +%% Returns affiliations without a version select_affs_by_room_id(MainHost, RoomID) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_select_affs_by_room_id, [RoomID]). @@ -387,8 +390,8 @@ get_aff_users({RoomU, RoomS} = RoomUS) -> {selected, [{Version, null, null, null}]} -> {ok, [], Version}; {selected, [{Version, _, _, _} | _] = Res} -> - AffUsers = [{{UserU, UserS}, aff_db2atom(Aff)} || {_, UserU, UserS, Aff} <- Res], - {ok, lists:sort(AffUsers), Version} + AffUsers = decode_affs_with_versions(Res), + {ok, AffUsers, Version} end. -spec modify_aff_users(RoomUS :: jid:simple_bare_jid(), @@ -413,20 +416,29 @@ get_info({RoomU, RoomS} = RoomUS) -> case select_room_id_and_version(MainHost, RoomU, RoomS) of {selected, [{RoomID, Version}]} -> {selected, AffUsersDB} = select_affs_by_room_id(MainHost, RoomID), - AffUsers = [{{UserU, UserS}, aff_db2atom(Aff)} || {UserU, UserS, Aff} <- AffUsersDB], - + AffUsers = decode_affs(AffUsersDB), {selected, ConfigDB} = mongoose_rdbms:sql_query( MainHost, mod_muc_light_db_rdbms_sql:select_config(RoomID)), {ok, Config} = mod_muc_light_room_config:apply_binary_kv( ConfigDB, [], mod_muc_light:config_schema(RoomS)), - {ok, Config, lists:sort(AffUsers), Version}; + {ok, Config, AffUsers, Version}; {selected, []} -> {error, not_exists} end. %% ------------------------ Conversions ------------------------ +decode_affs(AffUsersDB) -> + US2Aff = [{{UserU, UserS}, aff_db2atom(Aff)} + || {UserU, UserS, Aff} <- AffUsersDB], + lists:sort(US2Aff). + +decode_affs_with_versions(AffUsersDB) -> + US2Aff = [{{UserU, UserS}, aff_db2atom(Aff)} + || {_Version, UserU, UserS, Aff} <- AffUsersDB], + lists:sort(US2Aff). + -spec what_db2atom(binary() | pos_integer()) -> blocking_what(). what_db2atom(1) -> room; what_db2atom(2) -> user; @@ -575,8 +587,7 @@ modify_aff_users_transaction(MainHost, {RoomU, RoomS} = RoomUS, modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, CheckFun, PrevVersion, Version) -> {selected, AffUsersDB} = select_affs_by_room_id(MainHost, RoomID), - AffUsers = lists:sort( - [{{UserU, UserS}, aff_db2atom(Aff)} || {UserU, UserS, Aff} <- AffUsersDB]), + AffUsers = decode_affs(AffUsersDB), case mod_muc_light_utils:change_aff_users(AffUsers, AffUsersChanges) of {ok, NewAffUsers, AffUsersChanged, JoiningUsers, _LeavingUsers} -> case CheckFun(RoomUS, NewAffUsers) of From 678783fc5bf349e66b785f72424d1d2bf1b47098 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 14:08:18 +0100 Subject: [PATCH 73/92] User prepared insert_aff function --- src/muc_light/mod_muc_light_db_rdbms.erl | 23 +++++++++++++------- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 9 +------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index 9a2d12fb55..35bf32a916 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -145,6 +145,10 @@ prepare_affiliation_queries(Host) -> [room_id], <<"SELECT luser, lserver, aff " "FROM muc_light_occupants WHERE room_id = ?">>), + mongoose_rdbms:prepare(muc_light_insert_aff, muc_light_occupants, + [room_id, luser, lserver, aff], + <<"INSERT INTO muc_light_occupants (room_id, luser, lserver, aff)" + " VALUES(?, ?, ?, ?)">>), ok. %% ------------------------ Room SQL functions ------------------------ @@ -189,6 +193,11 @@ select_affs_by_room_id(MainHost, RoomID) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_select_affs_by_room_id, [RoomID]). +insert_aff(MainHost, RoomID, UserU, UserS, Aff) -> + DbAff = mod_muc_light_db_rdbms:aff_atom2db(Aff), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_insert_aff, [RoomID, UserU, UserS, DbAff]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), @@ -491,8 +500,7 @@ create_room_transaction(MainHost, {RoomU, RoomS}, Config, AffUsers, Version) -> {selected, [{RoomID}]} = select_room_id(MainHost, RoomU, RoomS), lists:foreach( fun({{UserU, UserS}, Aff}) -> - Query = mod_muc_light_db_rdbms_sql:insert_aff(RoomID, UserU, UserS, Aff), - {updated, _} = mongoose_rdbms:sql_query_t(Query) + {updated, _} = insert_aff(MainHost, RoomID, UserU, UserS, Aff) end, AffUsers), ConfigFields = mod_muc_light_room_config:to_binary_kv( Config, mod_muc_light:config_schema(RoomS)), @@ -592,7 +600,7 @@ modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, {ok, NewAffUsers, AffUsersChanged, JoiningUsers, _LeavingUsers} -> case CheckFun(RoomUS, NewAffUsers) of ok -> - apply_aff_users_transaction(RoomID, AffUsersChanged, JoiningUsers), + apply_aff_users_transaction(MainHost, RoomID, AffUsersChanged, JoiningUsers), {RoomU, RoomS} = RoomUS, {updated, _} = update_room_version(MainHost, RoomU, RoomS, Version), {ok, AffUsers, NewAffUsers, AffUsersChanged, PrevVersion}; @@ -603,10 +611,11 @@ modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, Error end. --spec apply_aff_users_transaction(RoomID :: binary(), +-spec apply_aff_users_transaction(MainHost :: jid:lserver(), + RoomID :: binary(), AffUsersChanges :: aff_users(), JoiningUsers :: [jid:simple_bare_jid()]) -> ok. -apply_aff_users_transaction(RoomID, AffUsersChanged, JoiningUsers) -> +apply_aff_users_transaction(MainHost, RoomID, AffUsersChanged, JoiningUsers) -> lists:foreach( fun({{UserU, UserS}, none}) -> {updated, _} = mongoose_rdbms:sql_query_t( @@ -614,9 +623,7 @@ apply_aff_users_transaction(RoomID, AffUsersChanged, JoiningUsers) -> ({{UserU, UserS} = UserUS, Aff}) -> case lists:member(UserUS, JoiningUsers) of true -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:insert_aff( - RoomID, UserU, UserS, Aff)); + {updated, _} = insert_aff(MainHost, RoomID, UserU, UserS, Aff); false -> {updated, _} = mongoose_rdbms:sql_query_t( mod_muc_light_db_rdbms_sql:update_aff( diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl index 0c6132dd58..7db79ad565 100644 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ b/src/muc_light/mod_muc_light_db_rdbms_sql.erl @@ -25,7 +25,7 @@ -include("mod_muc_light.hrl"). --export([insert_aff/4, update_aff/4, delete_affs/1, delete_aff/3]). +-export([update_aff/4, delete_affs/1, delete_aff/3]). -export([select_config/1, select_config/2, select_config/3, insert_config/3, update_config/3, delete_config/1]). -export([select_blocking/2, select_blocking_cnt/3, insert_blocking/4, @@ -37,13 +37,6 @@ %% Affiliations %%==================================================================== --spec insert_aff(RoomID :: integer() | binary(), UserU :: jid:luser(), - UserS :: jid:lserver(), Aff :: aff()) -> iolist(). -insert_aff(RoomID, UserU, UserS, Aff) -> - ["INSERT INTO muc_light_occupants (room_id, luser, lserver, aff)" - " VALUES(", bin(RoomID), ", ", ?ESC(UserU), ", ", ?ESC(UserS), ", ", - mod_muc_light_db_rdbms:aff_atom2db(Aff), ")"]. - -spec update_aff(RoomID :: integer() | binary(), UserU :: jid:luser(), UserS :: jid:lserver(), Aff :: aff()) -> iolist(). update_aff(RoomID, UserU, UserS, Aff) -> From 5062bf5921cd39d5eac4967a79fd218d97f6844d Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 14:11:37 +0100 Subject: [PATCH 74/92] User prepared update_aff function --- src/muc_light/mod_muc_light_db_rdbms.erl | 13 ++++++++++--- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 9 +-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index 35bf32a916..ce3410709d 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -149,6 +149,10 @@ prepare_affiliation_queries(Host) -> [room_id, luser, lserver, aff], <<"INSERT INTO muc_light_occupants (room_id, luser, lserver, aff)" " VALUES(?, ?, ?, ?)">>), + mongoose_rdbms:prepare(muc_light_update_aff, muc_light_occupants, + [aff, room_id, luser, lserver], + <<"UPDATE muc_light_occupants SET aff = ? " + "WHERE room_id = ? AND luser = ? AND lserver = ?">>), ok. %% ------------------------ Room SQL functions ------------------------ @@ -198,6 +202,11 @@ insert_aff(MainHost, RoomID, UserU, UserS, Aff) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_insert_aff, [RoomID, UserU, UserS, DbAff]). +update_aff(MainHost, RoomID, UserU, UserS, Aff) -> + DbAff = mod_muc_light_db_rdbms:aff_atom2db(Aff), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_update_aff, [DbAff, RoomID, UserU, UserS]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), @@ -625,9 +634,7 @@ apply_aff_users_transaction(MainHost, RoomID, AffUsersChanged, JoiningUsers) -> true -> {updated, _} = insert_aff(MainHost, RoomID, UserU, UserS, Aff); false -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:update_aff( - RoomID, UserU, UserS, Aff)) + {updated, _} = update_aff(MainHost, RoomID, UserU, UserS, Aff) end end, AffUsersChanged). diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl index 7db79ad565..fe2bad0047 100644 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ b/src/muc_light/mod_muc_light_db_rdbms_sql.erl @@ -25,7 +25,7 @@ -include("mod_muc_light.hrl"). --export([update_aff/4, delete_affs/1, delete_aff/3]). +-export([delete_affs/1, delete_aff/3]). -export([select_config/1, select_config/2, select_config/3, insert_config/3, update_config/3, delete_config/1]). -export([select_blocking/2, select_blocking_cnt/3, insert_blocking/4, @@ -37,13 +37,6 @@ %% Affiliations %%==================================================================== --spec update_aff(RoomID :: integer() | binary(), UserU :: jid:luser(), - UserS :: jid:lserver(), Aff :: aff()) -> iolist(). -update_aff(RoomID, UserU, UserS, Aff) -> - ["UPDATE muc_light_occupants SET aff = ", mod_muc_light_db_rdbms:aff_atom2db(Aff), - " WHERE room_id = ", bin(RoomID), " AND luser = ", ?ESC(UserU), - " AND lserver = ", ?ESC(UserS)]. - -spec delete_affs(RoomID :: integer() | binary()) -> iolist(). delete_affs(RoomID) -> ["DELETE FROM muc_light_occupants WHERE room_id = ", bin(RoomID)]. From 76c44aca94d67dbc421be1f7b952489fc3b5c519 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 14:17:07 +0100 Subject: [PATCH 75/92] Use prepared delete_aff/delete_affs functions --- src/muc_light/mod_muc_light_db_rdbms.erl | 21 ++++++++++++++++---- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 17 ---------------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index ce3410709d..37b241ad4f 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -153,6 +153,13 @@ prepare_affiliation_queries(Host) -> [aff, room_id, luser, lserver], <<"UPDATE muc_light_occupants SET aff = ? " "WHERE room_id = ? AND luser = ? AND lserver = ?">>), + mongoose_rdbms:prepare(muc_light_delete_affs, muc_light_occupants, + [room_id], + <<"DELETE FROM muc_light_occupants WHERE room_id = ?">>), + mongoose_rdbms:prepare(muc_light_delete_aff, muc_light_occupants, + [room_id, luser, lserver], + <<"DELETE FROM muc_light_occupants " + "WHERE room_id = ? AND luser = ? AND lserver = ?">>), ok. %% ------------------------ Room SQL functions ------------------------ @@ -207,6 +214,14 @@ update_aff(MainHost, RoomID, UserU, UserS, Aff) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_update_aff, [DbAff, RoomID, UserU, UserS]). +delete_affs(MainHost, RoomID) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_affs, [RoomID]). + +delete_aff(MainHost, RoomID, UserU, UserS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_aff, [RoomID, UserU, UserS]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), @@ -526,8 +541,7 @@ create_room_transaction(MainHost, {RoomU, RoomS}, Config, AffUsers, Version) -> destroy_room_transaction(MainHost, {RoomU, RoomS}) -> case select_room_id(MainHost, RoomU, RoomS) of {selected, [{RoomID}]} -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:delete_affs(RoomID)), + {updated, _} = delete_affs(MainHost, RoomID), {updated, _} = mongoose_rdbms:sql_query_t( mod_muc_light_db_rdbms_sql:delete_config(RoomID)), {updated, _} = delete_room(MainHost, RoomU, RoomS), @@ -627,8 +641,7 @@ modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, apply_aff_users_transaction(MainHost, RoomID, AffUsersChanged, JoiningUsers) -> lists:foreach( fun({{UserU, UserS}, none}) -> - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:delete_aff(RoomID, UserU, UserS)); + {updated, _} = delete_aff(MainHost, RoomID, UserU, UserS); ({{UserU, UserS} = UserUS, Aff}) -> case lists:member(UserUS, JoiningUsers) of true -> diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl index fe2bad0047..fc4e208e36 100644 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ b/src/muc_light/mod_muc_light_db_rdbms_sql.erl @@ -25,7 +25,6 @@ -include("mod_muc_light.hrl"). --export([delete_affs/1, delete_aff/3]). -export([select_config/1, select_config/2, select_config/3, insert_config/3, update_config/3, delete_config/1]). -export([select_blocking/2, select_blocking_cnt/3, insert_blocking/4, @@ -33,22 +32,6 @@ -define(ESC(T), mongoose_rdbms:use_escaped_string(mongoose_rdbms:escape_string(T))). -%%==================================================================== -%% Affiliations -%%==================================================================== - --spec delete_affs(RoomID :: integer() | binary()) -> iolist(). -delete_affs(RoomID) -> - ["DELETE FROM muc_light_occupants WHERE room_id = ", bin(RoomID)]. - --spec delete_aff(RoomID :: integer() | binary(), UserU :: jid:luser(), - UserS :: jid:lserver()) -> - iolist(). -delete_aff(RoomID, UserU, UserS) -> - ["DELETE FROM muc_light_occupants WHERE room_id = ", bin(RoomID), - " AND luser = ", ?ESC(UserU), - " AND lserver = ", ?ESC(UserS)]. - %%==================================================================== %% Config %%==================================================================== From 7f49c22b7ee6a20cc41d8eadab18013fb64ba22f Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 14:40:44 +0100 Subject: [PATCH 76/92] Make get_config function prepared --- src/muc_light/mod_muc_light_db.erl | 3 - src/muc_light/mod_muc_light_db_mnesia.erl | 14 ---- src/muc_light/mod_muc_light_db_rdbms.erl | 67 ++++++++++---------- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 18 +----- 4 files changed, 34 insertions(+), 68 deletions(-) diff --git a/src/muc_light/mod_muc_light_db.erl b/src/muc_light/mod_muc_light_db.erl index 5cc207492b..d6a86f5d96 100644 --- a/src/muc_light/mod_muc_light_db.erl +++ b/src/muc_light/mod_muc_light_db.erl @@ -55,9 +55,6 @@ -callback get_config(RoomUS :: jid:simple_bare_jid()) -> {ok, mod_muc_light_room_config:kv(), Version :: binary()} | {error, not_exists}. --callback get_config(RoomUS :: jid:simple_bare_jid(), Key :: atom()) -> - {ok, term(), Version :: binary()} | {error, not_exists | invalid_opt}. - -callback set_config(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), Version :: binary()) -> {ok, PrevVersion :: binary()} | {error, not_exists}. diff --git a/src/muc_light/mod_muc_light_db_mnesia.erl b/src/muc_light/mod_muc_light_db_mnesia.erl index 09eb86122c..6451245bd6 100644 --- a/src/muc_light/mod_muc_light_db_mnesia.erl +++ b/src/muc_light/mod_muc_light_db_mnesia.erl @@ -36,7 +36,6 @@ remove_user/2, get_config/1, - get_config/2, set_config/3, set_config/4, @@ -142,19 +141,6 @@ get_config(RoomUS) -> [#muc_light_room{ config = Config, version = Version }] -> {ok, Config, Version} end. --spec get_config(RoomUS :: jid:simple_bare_jid(), Key :: atom()) -> - {ok, term(), Version :: binary()} | {error, not_exists | invalid_opt}. -get_config(RoomUS, Option) -> - case get_config(RoomUS) of - {ok, Config, Version} -> - case lists:keyfind(Option, 1, Config) of - {_, Value} -> {ok, Value, Version}; - false -> {error, invalid_opt} - end; - Error -> - Error - end. - -spec set_config(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), Version :: binary()) -> diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index 37b241ad4f..2bcd9d2066 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -36,7 +36,6 @@ remove_user/2, get_config/1, - get_config/2, set_config/3, set_config/4, @@ -83,6 +82,13 @@ stop(_Host, _MUCHost) -> %% ------------------------ SQL ------------------------------------------- prepare_queries(Host) -> + prepare_cleaning_queries(Host), + prepare_room_queries(Host), + prepare_affiliation_queries(Host), + prepare_config_queries(Host), + ok. + +prepare_cleaning_queries(Host) -> mongoose_rdbms:prepare(muc_light_config_delete_all, muc_light_config, [], <<"DELETE FROM muc_light_config">>), mongoose_rdbms:prepare(muc_light_occupants_delete_all, muc_light_occupants, [], @@ -91,8 +97,6 @@ prepare_queries(Host) -> <<"DELETE FROM muc_light_rooms">>), mongoose_rdbms:prepare(muc_light_blocking_delete_all, muc_light_blocking, [], <<"DELETE FROM muc_light_blocking">>), - prepare_room_queries(Host), - prepare_affiliation_queries(Host), ok. prepare_room_queries(_Host) -> @@ -132,7 +136,7 @@ prepare_room_queries(_Host) -> " WHERE o.luser = ? AND o.lserver = ?">>), ok. -prepare_affiliation_queries(Host) -> +prepare_affiliation_queries(_Host) -> %% This query uses multiple table %% Also returns a room version mongoose_rdbms:prepare(muc_light_select_affs_by_us, muc_light_rooms, @@ -147,7 +151,8 @@ prepare_affiliation_queries(Host) -> "FROM muc_light_occupants WHERE room_id = ?">>), mongoose_rdbms:prepare(muc_light_insert_aff, muc_light_occupants, [room_id, luser, lserver, aff], - <<"INSERT INTO muc_light_occupants (room_id, luser, lserver, aff)" + <<"INSERT INTO muc_light_occupants" + " (room_id, luser, lserver, aff)" " VALUES(?, ?, ?, ?)">>), mongoose_rdbms:prepare(muc_light_update_aff, muc_light_occupants, [aff, room_id, luser, lserver], @@ -162,6 +167,18 @@ prepare_affiliation_queries(Host) -> "WHERE room_id = ? AND luser = ? AND lserver = ?">>), ok. +prepare_config_queries(_Host) -> + mongoose_rdbms:prepare(muc_light_select_config_by_room_id, muc_light_config, + [room_id], + <<"SELECT opt, val FROM muc_light_config WHERE room_id = ?">>), + mongoose_rdbms:prepare(muc_light_select_config_by_us, muc_light_rooms, + [luser, lserver], + <<"SELECT version, opt, val " + "FROM muc_light_rooms AS r " + "LEFT OUTER JOIN muc_light_config AS c ON r.id = c.room_id " + "WHERE r.luser = ? AND r.lserver = ?">>), + ok. + %% ------------------------ Room SQL functions ------------------------ select_room_id(MainHost, RoomU, RoomS) -> @@ -222,6 +239,15 @@ delete_aff(MainHost, RoomID, UserU, UserS) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_delete_aff, [RoomID, UserU, UserS]). +%% ------------------------ Config SQL functions --------------------------- +select_config_by_room_id(MainHost, RoomID) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_config_by_room_id, [RoomID]). + +select_config_by_us(MainHost, RoomU, RoomS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_config_by_us, [RoomU, RoomS]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), @@ -317,10 +343,7 @@ remove_user({_, UserS} = UserUS, Version) -> {ok, mod_muc_light_room_config:kv(), Version :: binary()} | {error, not_exists}. get_config({RoomU, RoomS} = RoomUS) -> MainHost = main_host(RoomUS), - - SQL = mod_muc_light_db_rdbms_sql:select_config(RoomU, RoomS), - {selected, Result} = mongoose_rdbms:sql_query(MainHost, SQL), - + {selected, Result} = select_config_by_us(MainHost, RoomU, RoomS), case Result of [] -> {error, not_exists}; @@ -333,29 +356,6 @@ get_config({RoomU, RoomS} = RoomUS) -> {ok, Config, Version} end. --spec get_config(RoomUS :: jid:simple_bare_jid(), Key :: atom()) -> - {ok, term(), Version :: binary()} | {error, not_exists | invalid_opt}. -get_config({RoomU, RoomS} = RoomUS, Key) -> - MainHost = main_host(RoomUS), - ConfigSchema = mod_muc_light:config_schema(RoomS), - {ok, KeyDB} = mod_muc_light_room_config:schema_reverse_find(Key, ConfigSchema), - - SQL = mod_muc_light_db_rdbms_sql:select_config(RoomU, RoomS, KeyDB), - {selected, Result} = mongoose_rdbms:sql_query(MainHost, SQL), - - case Result of - [] -> - {error, not_exists}; - [{_Version, null, null}] -> - {error, invalid_opt}; - [{Version, _, ValDB}] -> - RawConfig = [{KeyDB, ValDB}], - {ok, [{_, Val}]} = mod_muc_light_room_config:apply_binary_kv( - RawConfig, [], ConfigSchema), - {ok, Val, Version} - end. - - -spec set_config(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), Version :: binary()) -> {ok, PrevVersion :: binary()} | {error, not_exists}. @@ -450,8 +450,7 @@ get_info({RoomU, RoomS} = RoomUS) -> {selected, [{RoomID, Version}]} -> {selected, AffUsersDB} = select_affs_by_room_id(MainHost, RoomID), AffUsers = decode_affs(AffUsersDB), - {selected, ConfigDB} = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:select_config(RoomID)), + {selected, ConfigDB} = select_config_by_room_id(MainHost, RoomID), {ok, Config} = mod_muc_light_room_config:apply_binary_kv( ConfigDB, [], mod_muc_light:config_schema(RoomS)), diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl index fc4e208e36..712b0494c1 100644 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ b/src/muc_light/mod_muc_light_db_rdbms_sql.erl @@ -25,8 +25,7 @@ -include("mod_muc_light.hrl"). --export([select_config/1, select_config/2, select_config/3, insert_config/3, update_config/3, - delete_config/1]). +-export([insert_config/3, update_config/3, delete_config/1]). -export([select_blocking/2, select_blocking_cnt/3, insert_blocking/4, delete_blocking/4, delete_blocking/2]). @@ -36,21 +35,6 @@ %% Config %%==================================================================== --spec select_config(RoomID :: integer() | binary()) -> iolist(). -select_config(RoomID) -> - ["SELECT opt, val FROM muc_light_config WHERE room_id = ", bin(RoomID)]. - --spec select_config(RoomU :: jid:luser(), RoomS :: jid:lserver()) -> iolist(). -select_config(RoomU, RoomS) -> - ["SELECT version, opt, val", - " FROM muc_light_rooms AS r LEFT OUTER JOIN muc_light_config AS c ON r.id = c.room_id" - " WHERE r.luser = ", ?ESC(RoomU), " AND r.lserver = ", ?ESC(RoomS)]. - --spec select_config(RoomU :: jid:luser(), RoomS :: jid:lserver(), Key :: binary()) -> - iolist(). -select_config(RoomU, RoomS, Key) -> - [ select_config(RoomU, RoomS), " AND key = '", Key, "'" ]. - -spec insert_config(RoomID :: integer() | binary(), Key :: binary(), Val :: binary()) -> iolist(). insert_config(RoomID, Key, Val) -> ["INSERT INTO muc_light_config (room_id, opt, val)" From b51bff757607d196bc30896aba45ca4d934e1e7b Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 14:48:56 +0100 Subject: [PATCH 77/92] Use prepared functions for insert, update and delete room configs --- src/muc_light/mod_muc_light_db_rdbms.erl | 38 +++++++++++++++----- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 27 -------------- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index 2bcd9d2066..deb2e00006 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -121,7 +121,7 @@ prepare_room_queries(_Host) -> [luser, lserver], <<"DELETE FROM muc_light_rooms" " WHERE luser = ? AND lserver = ?">>), - %% This query uses multiple table + %% This query uses multiple tables mongoose_rdbms:prepare(muc_light_select_user_rooms, muc_light_occupants, [luser, lserver], <<"SELECT r.luser, r.lserver " @@ -137,7 +137,7 @@ prepare_room_queries(_Host) -> ok. prepare_affiliation_queries(_Host) -> - %% This query uses multiple table + %% This query uses multiple tables %% Also returns a room version mongoose_rdbms:prepare(muc_light_select_affs_by_us, muc_light_rooms, [luser, lserver], @@ -171,12 +171,24 @@ prepare_config_queries(_Host) -> mongoose_rdbms:prepare(muc_light_select_config_by_room_id, muc_light_config, [room_id], <<"SELECT opt, val FROM muc_light_config WHERE room_id = ?">>), + %% This query uses multiple tables mongoose_rdbms:prepare(muc_light_select_config_by_us, muc_light_rooms, [luser, lserver], <<"SELECT version, opt, val " "FROM muc_light_rooms AS r " "LEFT OUTER JOIN muc_light_config AS c ON r.id = c.room_id " "WHERE r.luser = ? AND r.lserver = ?">>), + mongoose_rdbms:prepare(muc_light_insert_config, muc_light_config, + [room_id, opt, val], + <<"INSERT INTO muc_light_config (room_id, opt, val)" + " VALUES(?, ?, ?)">>), + mongoose_rdbms:prepare(muc_light_update_config, muc_light_config, + [val, room_id, opt], + <<"UPDATE muc_light_config SET val = ? " + "WHERE room_id = ? AND opt = ?">>), + mongoose_rdbms:prepare(muc_light_delete_config, muc_light_config, + [room_id], + <<"DELETE FROM muc_light_config WHERE room_id = ?">>), ok. %% ------------------------ Room SQL functions ------------------------ @@ -248,6 +260,18 @@ select_config_by_us(MainHost, RoomU, RoomS) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_select_config_by_us, [RoomU, RoomS]). +insert_config(MainHost, RoomID, Key, Val) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_insert_config, [RoomID, Key, Val]). + +update_config(MainHost, RoomID, Key, Val) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_update_config, [Val, RoomID, Key]). + +delete_config(MainHost, RoomID) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_config, [RoomID]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), @@ -529,8 +553,7 @@ create_room_transaction(MainHost, {RoomU, RoomS}, Config, AffUsers, Version) -> Config, mod_muc_light:config_schema(RoomS)), lists:foreach( fun({Key, Val}) -> - Query = mod_muc_light_db_rdbms_sql:insert_config(RoomID, Key, Val), - {updated, _} = mongoose_rdbms:sql_query_t(Query) + {updated, _} = insert_config(MainHost, RoomID, Key, Val) end, ConfigFields), ok. @@ -541,8 +564,7 @@ destroy_room_transaction(MainHost, {RoomU, RoomS}) -> case select_room_id(MainHost, RoomU, RoomS) of {selected, [{RoomID}]} -> {updated, _} = delete_affs(MainHost, RoomID), - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:delete_config(RoomID)), + {updated, _} = delete_config(MainHost, RoomID), {updated, _} = delete_room(MainHost, RoomU, RoomS), ok; {selected, []} -> @@ -576,9 +598,7 @@ set_config_transaction({RoomU, RoomS} = RoomUS, ConfigChanges, Version) -> {updated, _} = update_room_version(MainHost, RoomU, RoomS, Version), lists:foreach( fun({Key, Val}) -> - {updated, _} - = mongoose_rdbms:sql_query( - MainHost, mod_muc_light_db_rdbms_sql:update_config(RoomID, Key, Val)) + {updated, _} = update_config(MainHost, RoomID, Key, Val) end, mod_muc_light_room_config:to_binary_kv( ConfigChanges, mod_muc_light:config_schema(RoomS))), {ok, PrevVersion}; diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl index 712b0494c1..bb78f6fdfe 100644 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ b/src/muc_light/mod_muc_light_db_rdbms_sql.erl @@ -25,30 +25,11 @@ -include("mod_muc_light.hrl"). --export([insert_config/3, update_config/3, delete_config/1]). -export([select_blocking/2, select_blocking_cnt/3, insert_blocking/4, delete_blocking/4, delete_blocking/2]). -define(ESC(T), mongoose_rdbms:use_escaped_string(mongoose_rdbms:escape_string(T))). -%%==================================================================== -%% Config -%%==================================================================== - --spec insert_config(RoomID :: integer() | binary(), Key :: binary(), Val :: binary()) -> iolist(). -insert_config(RoomID, Key, Val) -> - ["INSERT INTO muc_light_config (room_id, opt, val)" - " VALUES(", bin(RoomID), ", ", ?ESC(Key), ", ", ?ESC(Val), ")"]. - --spec update_config(RoomID :: integer() | binary(), Key :: binary(), Val :: binary()) -> iolist(). -update_config(RoomID, Key, Val) -> - ["UPDATE muc_light_config SET val = ", ?ESC(Val), - " WHERE room_id = ", bin(RoomID), " AND opt = ", ?ESC(Key)]. - --spec delete_config(RoomID :: integer() | binary()) -> iolist(). -delete_config(RoomID) -> - ["DELETE FROM muc_light_config WHERE room_id = ", bin(RoomID)]. - %%==================================================================== %% Blocking %%==================================================================== @@ -89,11 +70,3 @@ delete_blocking(LUser, LServer, What, Who) -> delete_blocking(UserU, UserS) -> ["DELETE FROM muc_light_blocking" " WHERE luser = ", ?ESC(UserU), " AND lserver = ", ?ESC(UserS)]. - -%%==================================================================== -%% Helpers -%%==================================================================== - --spec bin(integer() | binary()) -> binary(). -bin(Int) when is_integer(Int) -> integer_to_binary(Int); -bin(Bin) when is_binary(Bin) -> Bin. From d6f6bc3dd72f5a6539f8589a2d69fdb823c39ba3 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 18 Jan 2021 15:35:05 +0100 Subject: [PATCH 78/92] Use prepared queries for muc-light blocking --- src/muc_light/mod_muc_light_db_rdbms.erl | 109 ++++++++++++++++--- src/muc_light/mod_muc_light_db_rdbms_sql.erl | 72 ------------ 2 files changed, 91 insertions(+), 90 deletions(-) delete mode 100644 src/muc_light/mod_muc_light_db_rdbms_sql.erl diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index deb2e00006..3484e06e22 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -86,6 +86,7 @@ prepare_queries(Host) -> prepare_room_queries(Host), prepare_affiliation_queries(Host), prepare_config_queries(Host), + prepare_blocking_queries(Host), ok. prepare_cleaning_queries(Host) -> @@ -191,6 +192,36 @@ prepare_config_queries(_Host) -> <<"DELETE FROM muc_light_config WHERE room_id = ?">>), ok. +prepare_blocking_queries(_Host) -> + mongoose_rdbms:prepare(muc_light_select_blocking, muc_light_config, + [luser, lserver], + <<"SELECT what, who FROM muc_light_blocking " + "WHERE luser = ? AND lserver = ?">>), + mongoose_rdbms:prepare(muc_light_select_blocking_cnt, muc_light_config, + [luser, lserver, what, who], + <<"SELECT COUNT(*) FROM muc_light_blocking " + "WHERE luser = ? AND lserver = ? AND " + "what = ? AND who = ?">>), + mongoose_rdbms:prepare(muc_light_select_blocking_cnt2, muc_light_config, + [luser, lserver, what, who, what, who], + <<"SELECT COUNT(*) FROM muc_light_blocking " + "WHERE luser = ? AND lserver = ? AND " + "((what = ? AND who = ?) OR (what = ? AND who = ?))">>), + mongoose_rdbms:prepare(muc_light_insert_blocking, muc_light_blocking, + [luser, lserver, what, who], + <<"INSERT INTO muc_light_blocking" + " (luser, lserver, what, who)" + " VALUES (?, ?, ?, ?)">>), + mongoose_rdbms:prepare(muc_light_delete_blocking1, muc_light_blocking, + [luser, lserver, what, who], + <<"DELETE FROM muc_light_blocking " + "WHERE luser = ? AND lserver = ? AND what = ? AND who = ?">>), + mongoose_rdbms:prepare(muc_light_delete_blocking, muc_light_blocking, + [luser, lserver], + <<"DELETE FROM muc_light_blocking" + " WHERE luser = ? AND lserver = ?">>), + ok. + %% ------------------------ Room SQL functions ------------------------ select_room_id(MainHost, RoomU, RoomS) -> @@ -272,6 +303,45 @@ delete_config(MainHost, RoomID) -> mongoose_rdbms:execute_successfully( MainHost, muc_light_delete_config, [RoomID]). +%% ------------------------ Blocking SQL functions ------------------------- + +select_blocking(MainHost, LUser, LServer) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_blocking, [LUser, LServer]). + +select_blocking_cnt(MainHost, LUser, LServer, [{What, Who}]) -> + DbWhat = mod_muc_light_db_rdbms:what_atom2db(What), + DbWho = jid:to_binary(Who), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_blocking_cnt, + [LUser, LServer, DbWhat, DbWho]); +select_blocking_cnt(MainHost, LUser, LServer, [{What1, Who1}, {What2, Who2}]) -> + DbWhat1 = mod_muc_light_db_rdbms:what_atom2db(What1), + DbWhat2 = mod_muc_light_db_rdbms:what_atom2db(What2), + DbWho1 = jid:to_binary(Who1), + DbWho2 = jid:to_binary(Who2), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_select_blocking_cnt2, + [LUser, LServer, DbWhat1, DbWho1, DbWhat2, DbWho2]). + +insert_blocking(MainHost, LUser, LServer, What, Who) -> + DbWhat = mod_muc_light_db_rdbms:what_atom2db(What), + DbWho = jid:to_binary(Who), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_insert_blocking, + [LUser, LServer, DbWhat, DbWho]). + +delete_blocking1(MainHost, LUser, LServer, What, Who) -> + DbWhat = mod_muc_light_db_rdbms:what_atom2db(What), + DbWho = jid:to_binary(Who), + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_blocking1, + [LUser, LServer, DbWhat, DbWho]). + +delete_blocking(MainHost, UserU, UserS) -> + mongoose_rdbms:execute_successfully( + MainHost, muc_light_delete_blocking, [UserU, UserS]). + %% ------------------------ General room management ------------------------ -spec create_room(RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), @@ -402,9 +472,8 @@ set_config(RoomJID, Key, Val, Version) -> [blocking_item()]. get_blocking({LUser, LServer}, MUCServer) -> MainHost = main_host(MUCServer), - SQL = mod_muc_light_db_rdbms_sql:select_blocking(LUser, LServer), - {selected, WhatWhos} = mongoose_rdbms:sql_query(MainHost, SQL), - [ {what_db2atom(What), deny, jid:to_lus(jid:from_binary(Who))} || {What, Who} <- WhatWhos ]. + {selected, WhatWhos} = select_blocking(MainHost, LUser, LServer), + decode_blocking(WhatWhos). -spec get_blocking(UserUS :: jid:simple_bare_jid(), MUCServer :: jid:lserver(), @@ -412,8 +481,7 @@ get_blocking({LUser, LServer}, MUCServer) -> blocking_action(). get_blocking({LUser, LServer}, MUCServer, WhatWhos) -> MainHost = main_host(MUCServer), - SQL = mod_muc_light_db_rdbms_sql:select_blocking_cnt(LUser, LServer, WhatWhos), - {selected, [{Count}]} = mongoose_rdbms:sql_query(MainHost, SQL), + {selected, [{Count}]} = select_blocking_cnt(MainHost, LUser, LServer, WhatWhos), case mongoose_rdbms:result_to_integer(Count) of 0 -> allow; _ -> deny @@ -422,18 +490,20 @@ get_blocking({LUser, LServer}, MUCServer, WhatWhos) -> -spec set_blocking(UserUS :: jid:simple_bare_jid(), MUCServer :: jid:lserver(), BlockingItems :: [blocking_item()]) -> ok. -set_blocking(_UserUS, _MUCServer, []) -> +set_blocking(UserUS, MUCServer, BlockingItems) -> + MainHost = main_host(MUCServer), + set_blocking_loop(MainHost, UserUS, MUCServer, BlockingItems). + +set_blocking_loop(_MainHost, _UserUS, _MUCServer, []) -> ok; -set_blocking({LUser, LServer} = UserUS, MUCServer, [{What, deny, Who} | RBlockingItems]) -> - {updated, _} = - mongoose_rdbms:sql_query( - main_host(MUCServer), mod_muc_light_db_rdbms_sql:insert_blocking(LUser, LServer, What, Who)), - set_blocking(UserUS, MUCServer, RBlockingItems); -set_blocking({LUser, LServer} = UserUS, MUCServer, [{What, allow, Who} | RBlockingItems]) -> - {updated, _} = - mongoose_rdbms:sql_query( - main_host(MUCServer), mod_muc_light_db_rdbms_sql:delete_blocking(LUser, LServer, What, Who)), - set_blocking(UserUS, MUCServer, RBlockingItems). +set_blocking_loop(MainHost, {LUser, LServer} = UserUS, MUCServer, + [{What, deny, Who} | RBlockingItems]) -> + {updated, _} = insert_blocking(MainHost, LUser, LServer, What, Who), + set_blocking_loop(MainHost, UserUS, MUCServer, RBlockingItems); +set_blocking_loop(MainHost, {LUser, LServer} = UserUS, MUCServer, + [{What, allow, Who} | RBlockingItems]) -> + {updated, _} = delete_blocking1(MainHost, LUser, LServer, What, Who), + set_blocking_loop(MainHost, UserUS, MUCServer, RBlockingItems). %% ------------------------ Affiliations manipulation ------------------------ @@ -495,6 +565,10 @@ decode_affs_with_versions(AffUsersDB) -> || {_Version, UserU, UserS, Aff} <- AffUsersDB], lists:sort(US2Aff). +decode_blocking(WhatWhos) -> + [ {what_db2atom(What), deny, jid:to_lus(jid:from_binary(Who))} + || {What, Who} <- WhatWhos ]. + -spec what_db2atom(binary() | pos_integer()) -> blocking_what(). what_db2atom(1) -> room; what_db2atom(2) -> user; @@ -577,8 +651,7 @@ destroy_room_transaction(MainHost, {RoomU, RoomS}) -> mod_muc_light_db:remove_user_return(). remove_user_transaction(MainHost, {UserU, UserS} = UserUS, Version) -> Rooms = get_user_rooms(UserUS, undefined), - {updated, _} = mongoose_rdbms:sql_query_t( - mod_muc_light_db_rdbms_sql:delete_blocking(UserU, UserS)), + {updated, _} = delete_blocking(MainHost, UserU, UserS), lists:map( fun(RoomUS) -> {RoomUS, modify_aff_users_transaction(MainHost, diff --git a/src/muc_light/mod_muc_light_db_rdbms_sql.erl b/src/muc_light/mod_muc_light_db_rdbms_sql.erl deleted file mode 100644 index bb78f6fdfe..0000000000 --- a/src/muc_light/mod_muc_light_db_rdbms_sql.erl +++ /dev/null @@ -1,72 +0,0 @@ -%%%---------------------------------------------------------------------- -%%% File : mod_muc_light_db_rdbms_sql.erl -%%% Author : Piotr Nosek -%%% Purpose : RDBMS backend queries for mod_muc_light -%%% Created : 29 Nov 2016 by Piotr Nosek -%%% -%%% This program is free software; you can redistribute it and/or -%%% modify it under the terms of the GNU General Public License as -%%% published by the Free Software Foundation; either version 2 of the -%%% License, or (at your option) any later version. -%%% -%%% This program is distributed in the hope that it will be useful, -%%% but WITHOUT ANY WARRANTY; without even the implied warranty of -%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -%%% General Public License for more details. -%%% -%%% You should have received a copy of the GNU General Public License -%%% along with this program; if not, write to the Free Software -%%% Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -%%% -%%%---------------------------------------------------------------------- - --module(mod_muc_light_db_rdbms_sql). --author('piotr.nosek@erlang-solutions.com'). - --include("mod_muc_light.hrl"). - --export([select_blocking/2, select_blocking_cnt/3, insert_blocking/4, - delete_blocking/4, delete_blocking/2]). - --define(ESC(T), mongoose_rdbms:use_escaped_string(mongoose_rdbms:escape_string(T))). - -%%==================================================================== -%% Blocking -%%==================================================================== - --spec select_blocking(LUser :: jid:luser(), LServer :: jid:lserver()) -> iolist(). -select_blocking(LUser, LServer) -> - ["SELECT what, who FROM muc_light_blocking WHERE luser = ", ?ESC(LUser), - " AND lserver = ", ?ESC(LServer)]. - --spec select_blocking_cnt(LUser :: jid:luser(), LServer :: jid:lserver(), - WhatWhos :: [{blocking_who(), jid:simple_bare_jid()}]) -> iolist(). -select_blocking_cnt(LUser, LServer, WhatWhos) -> - [ _ | WhatWhosWhere ] = lists:flatmap( - fun({What, Who}) -> - [" OR ", "(what = ", mod_muc_light_db_rdbms:what_atom2db(What), - " AND who = ", ?ESC(jid:to_binary(Who)), ")"] end, - WhatWhos), - ["SELECT COUNT(*) FROM muc_light_blocking WHERE luser = ", ?ESC(LUser), - " AND lserver = ", ?ESC(LServer), - " AND (", WhatWhosWhere, ")"]. - --spec insert_blocking(LUser :: jid:luser(), LServer :: jid:lserver(), - What :: blocking_what(), Who :: blocking_who()) -> iolist(). -insert_blocking(LUser, LServer, What, Who) -> - ["INSERT INTO muc_light_blocking (luser, lserver, what, who)" - " VALUES (", ?ESC(LUser), ", ", ?ESC(LServer), ", ", - mod_muc_light_db_rdbms:what_atom2db(What), ", ", ?ESC(jid:to_binary(Who)), ")"]. - --spec delete_blocking(LUser :: jid:luser(), LServer :: jid:lserver(), - What :: blocking_what(), Who :: blocking_who()) -> iolist(). -delete_blocking(LUser, LServer, What, Who) -> - ["DELETE FROM muc_light_blocking WHERE luser = ", ?ESC(LUser), - " AND lserver = ", ?ESC(LServer), - " AND what = ", mod_muc_light_db_rdbms:what_atom2db(What), - " AND who = ", ?ESC(jid:to_binary(Who))]. - --spec delete_blocking(UserU :: jid:luser(), UserS :: jid:lserver()) -> iolist(). -delete_blocking(UserU, UserS) -> - ["DELETE FROM muc_light_blocking" - " WHERE luser = ", ?ESC(UserU), " AND lserver = ", ?ESC(UserS)]. From ca753d07dbf46398a7299097ed3f2ea33187d8a2 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 19 Jan 2021 13:06:25 +0100 Subject: [PATCH 79/92] Fix types for affiliations --- src/muc_light/mod_muc_light_db_rdbms.erl | 32 ++++++++++-------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index 3484e06e22..c99771ba97 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -49,12 +49,6 @@ get_info/1 ]). -%% Conversions --export([ - what_db2atom/1, what_atom2db/1, - aff_db2atom/1, aff_atom2db/1 - ]). - %% Extra API for testing -export([ force_clear/0 @@ -265,12 +259,12 @@ select_affs_by_room_id(MainHost, RoomID) -> MainHost, muc_light_select_affs_by_room_id, [RoomID]). insert_aff(MainHost, RoomID, UserU, UserS, Aff) -> - DbAff = mod_muc_light_db_rdbms:aff_atom2db(Aff), + DbAff = aff_atom2db(Aff), mongoose_rdbms:execute_successfully( MainHost, muc_light_insert_aff, [RoomID, UserU, UserS, DbAff]). update_aff(MainHost, RoomID, UserU, UserS, Aff) -> - DbAff = mod_muc_light_db_rdbms:aff_atom2db(Aff), + DbAff = aff_atom2db(Aff), mongoose_rdbms:execute_successfully( MainHost, muc_light_update_aff, [DbAff, RoomID, UserU, UserS]). @@ -310,14 +304,14 @@ select_blocking(MainHost, LUser, LServer) -> MainHost, muc_light_select_blocking, [LUser, LServer]). select_blocking_cnt(MainHost, LUser, LServer, [{What, Who}]) -> - DbWhat = mod_muc_light_db_rdbms:what_atom2db(What), + DbWhat = what_atom2db(What), DbWho = jid:to_binary(Who), mongoose_rdbms:execute_successfully( MainHost, muc_light_select_blocking_cnt, [LUser, LServer, DbWhat, DbWho]); select_blocking_cnt(MainHost, LUser, LServer, [{What1, Who1}, {What2, Who2}]) -> - DbWhat1 = mod_muc_light_db_rdbms:what_atom2db(What1), - DbWhat2 = mod_muc_light_db_rdbms:what_atom2db(What2), + DbWhat1 = what_atom2db(What1), + DbWhat2 = what_atom2db(What2), DbWho1 = jid:to_binary(Who1), DbWho2 = jid:to_binary(Who2), mongoose_rdbms:execute_successfully( @@ -325,14 +319,14 @@ select_blocking_cnt(MainHost, LUser, LServer, [{What1, Who1}, {What2, Who2}]) -> [LUser, LServer, DbWhat1, DbWho1, DbWhat2, DbWho2]). insert_blocking(MainHost, LUser, LServer, What, Who) -> - DbWhat = mod_muc_light_db_rdbms:what_atom2db(What), + DbWhat = what_atom2db(What), DbWho = jid:to_binary(Who), mongoose_rdbms:execute_successfully( MainHost, muc_light_insert_blocking, [LUser, LServer, DbWhat, DbWho]). delete_blocking1(MainHost, LUser, LServer, What, Who) -> - DbWhat = mod_muc_light_db_rdbms:what_atom2db(What), + DbWhat = what_atom2db(What), DbWho = jid:to_binary(Who), mongoose_rdbms:execute_successfully( MainHost, muc_light_delete_blocking1, @@ -574,13 +568,13 @@ what_db2atom(1) -> room; what_db2atom(2) -> user; what_db2atom(Bin) -> what_db2atom(mongoose_rdbms:result_to_integer(Bin)). --spec what_atom2db(blocking_what()) -> string(). -what_atom2db(room) -> "1"; -what_atom2db(user) -> "2". +-spec what_atom2db(blocking_what()) -> non_neg_integer(). +what_atom2db(room) -> 1; +what_atom2db(user) -> 2. --spec aff_atom2db(aff()) -> string(). -aff_atom2db(owner) -> "1"; -aff_atom2db(member) -> "2". +-spec aff_atom2db(aff()) -> non_neg_integer(). +aff_atom2db(owner) -> 1; +aff_atom2db(member) -> 2. -spec aff_db2atom(binary() | pos_integer()) -> aff(). aff_db2atom(1) -> owner; From e3c772eb78955f35f87fa486490af7c4e7acd938 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 19 Jan 2021 17:41:39 +0100 Subject: [PATCH 80/92] Fix muc-light queries for ODBC --- src/muc_light/mod_muc_light_db_rdbms.erl | 28 ++++++++++++++---------- src/rdbms/mongoose_rdbms_odbc.erl | 21 ++++++++++++++++-- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index c99771ba97..edc3d6085f 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -54,6 +54,8 @@ force_clear/0 ]). +-type room_id() :: non_neg_integer(). + -include("mongoose.hrl"). -include("jlib.hrl"). -include("mod_muc_light.hrl"). @@ -187,16 +189,16 @@ prepare_config_queries(_Host) -> ok. prepare_blocking_queries(_Host) -> - mongoose_rdbms:prepare(muc_light_select_blocking, muc_light_config, + mongoose_rdbms:prepare(muc_light_select_blocking, muc_light_blocking, [luser, lserver], <<"SELECT what, who FROM muc_light_blocking " "WHERE luser = ? AND lserver = ?">>), - mongoose_rdbms:prepare(muc_light_select_blocking_cnt, muc_light_config, + mongoose_rdbms:prepare(muc_light_select_blocking_cnt, muc_light_blocking, [luser, lserver, what, who], <<"SELECT COUNT(*) FROM muc_light_blocking " "WHERE luser = ? AND lserver = ? AND " "what = ? AND who = ?">>), - mongoose_rdbms:prepare(muc_light_select_blocking_cnt2, muc_light_config, + mongoose_rdbms:prepare(muc_light_select_blocking_cnt2, muc_light_blocking, [luser, lserver, what, who, what, who], <<"SELECT COUNT(*) FROM muc_light_blocking " "WHERE luser = ? AND lserver = ? AND " @@ -471,7 +473,7 @@ get_blocking({LUser, LServer}, MUCServer) -> -spec get_blocking(UserUS :: jid:simple_bare_jid(), MUCServer :: jid:lserver(), - WhatWhos :: [{blocking_who(), jid:simple_bare_jid()}]) -> + WhatWhos :: [{blocking_what(), jid:simple_bare_jid()}]) -> blocking_action(). get_blocking({LUser, LServer}, MUCServer, WhatWhos) -> MainHost = main_host(MUCServer), @@ -535,7 +537,8 @@ modify_aff_users(RoomUS, AffUsersChanges, ExternalCheck, Version) -> get_info({RoomU, RoomS} = RoomUS) -> MainHost = main_host(RoomUS), case select_room_id_and_version(MainHost, RoomU, RoomS) of - {selected, [{RoomID, Version}]} -> + {selected, [{DbRoomID, Version}]} -> + RoomID = mongoose_rdbms:result_to_integer(DbRoomID), {selected, AffUsersDB} = select_affs_by_room_id(MainHost, RoomID), AffUsers = decode_affs(AffUsersDB), {selected, ConfigDB} = select_config_by_room_id(MainHost, RoomID), @@ -612,7 +615,7 @@ force_clear() -> ok | {error, exists}. create_room_transaction(MainHost, {RoomU, RoomS}, Config, AffUsers, Version) -> insert_room(MainHost, RoomU, RoomS, Version), - {selected, [{RoomID}]} = select_room_id(MainHost, RoomU, RoomS), + RoomID = mongoose_rdbms:selected_to_integer(select_room_id(MainHost, RoomU, RoomS)), lists:foreach( fun({{UserU, UserS}, Aff}) -> {updated, _} = insert_aff(MainHost, RoomID, UserU, UserS, Aff) @@ -630,7 +633,8 @@ create_room_transaction(MainHost, {RoomU, RoomS}, Config, AffUsers, Version) -> ok | {error, not_exists}. destroy_room_transaction(MainHost, {RoomU, RoomS}) -> case select_room_id(MainHost, RoomU, RoomS) of - {selected, [{RoomID}]} -> + {selected, [{DbRoomID}]} -> + RoomID = mongoose_rdbms:result_to_integer(DbRoomID), {updated, _} = delete_affs(MainHost, RoomID), {updated, _} = delete_config(MainHost, RoomID), {updated, _} = delete_room(MainHost, RoomU, RoomS), @@ -661,7 +665,8 @@ remove_user_transaction(MainHost, {UserU, UserS} = UserUS, Version) -> set_config_transaction({RoomU, RoomS} = RoomUS, ConfigChanges, Version) -> MainHost = main_host(RoomUS), case select_room_id_and_version(MainHost, RoomU, RoomS) of - {selected, [{RoomID, PrevVersion}]} -> + {selected, [{DbRoomID, PrevVersion}]} -> + RoomID = mongoose_rdbms:result_to_integer(DbRoomID), {updated, _} = update_room_version(MainHost, RoomU, RoomS, Version), lists:foreach( fun({Key, Val}) -> @@ -686,7 +691,8 @@ set_config_transaction({RoomU, RoomS} = RoomUS, ConfigChanges, Version) -> modify_aff_users_transaction(MainHost, {RoomU, RoomS} = RoomUS, AffUsersChanges, CheckFun, Version) -> case select_room_id_and_version(MainHost, RoomU, RoomS) of - {selected, [{RoomID, PrevVersion}]} -> + {selected, [{DbRoomID, PrevVersion}]} -> + RoomID = mongoose_rdbms:result_to_integer(DbRoomID), modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, CheckFun, PrevVersion, Version); {selected, []} -> @@ -695,7 +701,7 @@ modify_aff_users_transaction(MainHost, {RoomU, RoomS} = RoomUS, -spec modify_aff_users_transaction(MainHost :: jid:lserver(), RoomUS :: jid:simple_bare_jid(), - RoomID :: binary(), + RoomID :: room_id(), AffUsersChanges :: aff_users(), CheckFun :: external_check_fun(), PrevVersion :: binary(), @@ -721,7 +727,7 @@ modify_aff_users_transaction(MainHost, RoomUS, RoomID, AffUsersChanges, end. -spec apply_aff_users_transaction(MainHost :: jid:lserver(), - RoomID :: binary(), + RoomID :: room_id(), AffUsersChanges :: aff_users(), JoiningUsers :: [jid:simple_bare_jid()]) -> ok. apply_aff_users_transaction(MainHost, RoomID, AffUsersChanged, JoiningUsers) -> diff --git a/src/rdbms/mongoose_rdbms_odbc.erl b/src/rdbms/mongoose_rdbms_odbc.erl index 764cde654e..1aa45e0e13 100644 --- a/src/rdbms/mongoose_rdbms_odbc.erl +++ b/src/rdbms/mongoose_rdbms_odbc.erl @@ -75,7 +75,16 @@ query(Connection, Query, Timeout) -> -spec prepare(Connection :: term(), Name :: atom(), Table :: binary(), Fields :: [binary()], Statement :: iodata()) -> {ok, {binary(), [fun((term()) -> tuple())]}}. -prepare(Connection, _Name, Table, Fields, Statement) -> +prepare(Connection, Name, Table, Fields, Statement) -> + try prepare2(Connection, Table, Fields, Statement) + catch Class:Reason:Stacktrace -> + ?LOG_ERROR(#{what => prepare_failed, + statement_name => Name, sql_query => Statement, + class => Class, reason => Reason, stacktrace => Stacktrace}), + erlang:raise(Class, Reason, Stacktrace) + end. + +prepare2(Connection, Table, Fields, Statement) -> {ok, TableDesc} = eodbc:describe_table(Connection, unicode:characters_to_list(Table)), ServerType = server_type(), ParamMappers = [field_name_to_mapper(ServerType, TableDesc, Field) || Field <- Fields], @@ -156,7 +165,7 @@ field_name_to_mapper(_ServerType, _TableDesc, <<"limit">>) -> field_name_to_mapper(_ServerType, _TableDesc, <<"offset">>) -> fun(P) -> {sql_integer, [P]} end; field_name_to_mapper(_ServerType, TableDesc, FieldName) -> - {_, ODBCType} = lists:keyfind(unicode:characters_to_list(FieldName), 1, TableDesc), + {_, ODBCType} = find_key(unicode:characters_to_list(FieldName), TableDesc), case simple_type(just_type(ODBCType)) of binary -> fun(P) -> binary_mapper(P) end; @@ -168,6 +177,14 @@ field_name_to_mapper(_ServerType, TableDesc, FieldName) -> fun(P) -> {ODBCType, [P]} end end. +find_key(Key, List) -> + case lists:keyfind(Key, 1, List) of + false -> + error(#{what => find_key_failed, key => Key, list => List}); + Tuple -> + Tuple + end. + unicode_mapper(P) -> Utf16 = unicode_characters_to_binary(iolist_to_binary(P), utf8, {utf16, little}), Len = byte_size(Utf16) div 2, From c6699f5430675ca078ad7742f204f02f027b219c Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 26 Jan 2021 17:52:42 +0100 Subject: [PATCH 81/92] Fix comment and typespec based on review --- src/muc_light/mod_muc_light_db_rdbms.erl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index edc3d6085f..2bbe7a4f3b 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -118,7 +118,7 @@ prepare_room_queries(_Host) -> [luser, lserver], <<"DELETE FROM muc_light_rooms" " WHERE luser = ? AND lserver = ?">>), - %% This query uses multiple tables + %% These queries use multiple tables mongoose_rdbms:prepare(muc_light_select_user_rooms, muc_light_occupants, [luser, lserver], <<"SELECT r.luser, r.lserver " @@ -611,8 +611,7 @@ force_clear() -> RoomUS :: jid:simple_bare_jid(), Config :: mod_muc_light_room_config:kv(), AffUsers :: aff_users(), - Version :: binary()) -> - ok | {error, exists}. + Version :: binary()) -> ok. create_room_transaction(MainHost, {RoomU, RoomS}, Config, AffUsers, Version) -> insert_room(MainHost, RoomU, RoomS, Version), RoomID = mongoose_rdbms:selected_to_integer(select_room_id(MainHost, RoomU, RoomS)), From b8f24ecbd81e76bf06fd05b69e1e0c784f575368 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 26 Jan 2021 18:03:22 +0100 Subject: [PATCH 82/92] Fix indention in mod_muc_light_db_rdbms --- src/muc_light/mod_muc_light_db_rdbms.erl | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index 2bbe7a4f3b..b3b1288568 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -85,7 +85,7 @@ prepare_queries(Host) -> prepare_blocking_queries(Host), ok. -prepare_cleaning_queries(Host) -> +prepare_cleaning_queries(_Host) -> mongoose_rdbms:prepare(muc_light_config_delete_all, muc_light_config, [], <<"DELETE FROM muc_light_config">>), mongoose_rdbms:prepare(muc_light_occupants_delete_all, muc_light_occupants, [], @@ -615,17 +615,13 @@ force_clear() -> create_room_transaction(MainHost, {RoomU, RoomS}, Config, AffUsers, Version) -> insert_room(MainHost, RoomU, RoomS, Version), RoomID = mongoose_rdbms:selected_to_integer(select_room_id(MainHost, RoomU, RoomS)), - lists:foreach( - fun({{UserU, UserS}, Aff}) -> - {updated, _} = insert_aff(MainHost, RoomID, UserU, UserS, Aff) - end, AffUsers), - ConfigFields = mod_muc_light_room_config:to_binary_kv( - Config, mod_muc_light:config_schema(RoomS)), - lists:foreach( - fun({Key, Val}) -> - {updated, _} = insert_config(MainHost, RoomID, Key, Val) - end, ConfigFields), - ok. + Schema = mod_muc_light:config_schema(RoomS), + ConfigFields = mod_muc_light_room_config:to_binary_kv(Config, Schema), + [insert_aff(MainHost, RoomID, UserU, UserS, Aff) + || {{UserU, UserS}, Aff} <- AffUsers], + [insert_config(MainHost, RoomID, Key, Val) + || {Key, Val} <- ConfigFields], + ok. -spec destroy_room_transaction(MainHost :: jid:lserver(), RoomUS :: jid:simple_bare_jid()) -> From aa5ca5cd5b5f8351599d9160aef1d82ab2d16019 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 27 Jan 2021 09:58:40 +0100 Subject: [PATCH 83/92] Replace find_key with field_to_odbc_type --- src/rdbms/mongoose_rdbms_odbc.erl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/rdbms/mongoose_rdbms_odbc.erl b/src/rdbms/mongoose_rdbms_odbc.erl index 1aa45e0e13..f7d4012744 100644 --- a/src/rdbms/mongoose_rdbms_odbc.erl +++ b/src/rdbms/mongoose_rdbms_odbc.erl @@ -165,7 +165,7 @@ field_name_to_mapper(_ServerType, _TableDesc, <<"limit">>) -> field_name_to_mapper(_ServerType, _TableDesc, <<"offset">>) -> fun(P) -> {sql_integer, [P]} end; field_name_to_mapper(_ServerType, TableDesc, FieldName) -> - {_, ODBCType} = find_key(unicode:characters_to_list(FieldName), TableDesc), + ODBCType = field_to_odbc_type(unicode:characters_to_list(FieldName), TableDesc), case simple_type(just_type(ODBCType)) of binary -> fun(P) -> binary_mapper(P) end; @@ -177,12 +177,14 @@ field_name_to_mapper(_ServerType, TableDesc, FieldName) -> fun(P) -> {ODBCType, [P]} end end. -find_key(Key, List) -> - case lists:keyfind(Key, 1, List) of +field_to_odbc_type(FieldName, TableDesc) -> + case lists:keyfind(FieldName, 1, TableDesc) of false -> - error(#{what => find_key_failed, key => Key, list => List}); - Tuple -> - Tuple + ?LOG_ERROR(#{what => field_to_odbc_type_failed, + field => FieldName, table_desc => TableDesc}), + error(field_to_odbc_type_failed); + {_, ODBCType} -> + ODBCType end. unicode_mapper(P) -> From 03db0fb46d2b287a5cf4620b21976d9b453e131b Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 27 Jan 2021 09:59:15 +0100 Subject: [PATCH 84/92] Ensure we match everything in create_room_transaction LCs --- src/muc_light/mod_muc_light_db_rdbms.erl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/muc_light/mod_muc_light_db_rdbms.erl b/src/muc_light/mod_muc_light_db_rdbms.erl index b3b1288568..ff90bf56a9 100644 --- a/src/muc_light/mod_muc_light_db_rdbms.erl +++ b/src/muc_light/mod_muc_light_db_rdbms.erl @@ -617,12 +617,16 @@ create_room_transaction(MainHost, {RoomU, RoomS}, Config, AffUsers, Version) -> RoomID = mongoose_rdbms:selected_to_integer(select_room_id(MainHost, RoomU, RoomS)), Schema = mod_muc_light:config_schema(RoomS), ConfigFields = mod_muc_light_room_config:to_binary_kv(Config, Schema), - [insert_aff(MainHost, RoomID, UserU, UserS, Aff) - || {{UserU, UserS}, Aff} <- AffUsers], - [insert_config(MainHost, RoomID, Key, Val) - || {Key, Val} <- ConfigFields], + [insert_aff_tuple(MainHost, RoomID, AffUser) || AffUser <- AffUsers], + [insert_config_kv(MainHost, RoomID, KV) || KV <- ConfigFields], ok. +insert_aff_tuple(MainHost, RoomID, {{UserU, UserS}, Aff}) -> + insert_aff(MainHost, RoomID, UserU, UserS, Aff). + +insert_config_kv(MainHost, RoomID, {Key, Val}) -> + insert_config(MainHost, RoomID, Key, Val). + -spec destroy_room_transaction(MainHost :: jid:lserver(), RoomUS :: jid:simple_bare_jid()) -> ok | {error, not_exists}. From e78ad859ddfbbd396961bcb795b019610f15d1d9 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 1 Feb 2021 11:06:45 +0100 Subject: [PATCH 85/92] Use prepared queries for mod_last --- src/mod_last_rdbms.erl | 82 ++++++++++++++++++++++--------------- src/rdbms/rdbms_queries.erl | 28 ------------- 2 files changed, 48 insertions(+), 62 deletions(-) diff --git a/src/mod_last_rdbms.erl b/src/mod_last_rdbms.erl index bb5b8cdf15..96ae404b58 100644 --- a/src/mod_last_rdbms.erl +++ b/src/mod_last_rdbms.erl @@ -25,52 +25,66 @@ remove_user/2]). -spec init(jid:server(), list()) -> ok. -init(_Host, _Opts) -> +init(Host, _Opts) -> + prepare_queries(Host), ok. +%% Prepared query functions +prepare_queries(Host) -> + mongoose_rdbms:prepare(last_select, last, [username], + <<"SELECT seconds, state FROM last WHERE username=?">>), + mongoose_rdbms:prepare(last_count_active, last, [seconds], + <<"SELECT COUNT(*) FROM last WHERE seconds > ?">>), + mongoose_rdbms:prepare(last_delete, last, [username], + <<"delete from last where username=?">>), + rdbms_queries:prepare_upsert(Host, last_upsert, last, + [<<"username">>, <<"seconds">>, <<"state">>], + [<<"seconds">>, <<"state">>], + [<<"username">>]). + +execute_get_last(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, last_select, [LUser]). + +execute_count_active_users(LServer, Seconds) -> + mongoose_rdbms:execute_successfully(LServer, last_count_active, [Seconds]). + +execute_remove_user(LServer, LUser) -> + mongoose_rdbms:execute_successfully(LServer, last_delete, [LUser]). + +execute_upsert_last(Host, LUser, Seconds, State) -> + InsertParams = [LUser, Seconds, State], + UpdateParams = [Seconds, State], + UniqueKeyValues = [LUser], + rdbms_queries:execute_upsert(Host, last_upsert, InsertParams, UpdateParams, UniqueKeyValues). + +%% API functions -spec get_last(jid:luser(), jid:lserver()) -> - {ok, non_neg_integer(), binary()} | {error, term()} | not_found. + {ok, non_neg_integer(), binary()} | not_found. get_last(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - case catch rdbms_queries:get_last(LServer, Username) of - {selected, []} -> - not_found; - {selected, [{STimeStamp, Status}]} -> - case catch mongoose_rdbms:result_to_integer(STimeStamp) of - TimeStamp when is_integer(TimeStamp) -> - {ok, TimeStamp, Status}; - Reason -> - {error, {invalid_timestamp, Reason}} - end; - Reason -> {error, {invalid_result, Reason}} - end. + Result = execute_get_last(LServer, LUser), + decode_last_result(Result). -spec count_active_users(jid:lserver(), non_neg_integer()) -> non_neg_integer(). -count_active_users(LServer, TimeStamp) -> - TimeStampBin = integer_to_binary(TimeStamp), - WhereClause = <<"where seconds > ", TimeStampBin/binary >>, - case rdbms_queries:count_records_where(LServer, <<"last">>, WhereClause) of - {selected, [{Count}]} -> - mongoose_rdbms:result_to_integer(Count); - _ -> - 0 - end. +count_active_users(LServer, Seconds) -> + Result = execute_count_active_users(LServer, Seconds), + mongoose_rdbms:selected_to_integer(Result). -spec set_last_info(jid:luser(), jid:lserver(), - non_neg_integer(), binary()) -> - ok | {error, term()}. -set_last_info(LUser, LServer, TimeStamp, Status) -> - Username = mongoose_rdbms:escape_string(LUser), - Seconds = mongoose_rdbms:escape_integer(TimeStamp), - State = mongoose_rdbms:escape_string(Status), - wrap_rdbms_result(rdbms_queries:set_last_t(LServer, Username, Seconds, State)). + non_neg_integer(), binary()) -> ok | {error, term()}. +set_last_info(LUser, LServer, Seconds, State) -> + wrap_rdbms_result(execute_upsert_last(LServer, LUser, Seconds, State)). -spec remove_user(jid:luser(), jid:lserver()) -> ok | {error, term()}. remove_user(LUser, LServer) -> - Username = mongoose_rdbms:escape_string(LUser), - wrap_rdbms_result(rdbms_queries:del_last(LServer, Username)). + wrap_rdbms_result(execute_remove_user(LServer, LUser)). + +%% Helper functions +decode_last_result({selected, []}) -> + not_found; +decode_last_result({selected, [{DbSeconds, State}]}) -> + Seconds = mongoose_rdbms:result_to_integer(DbSeconds), + {ok, Seconds, State}. -spec wrap_rdbms_result({error, term()} | any()) -> ok | {error, term()}. wrap_rdbms_result({error, _} = Error) -> Error; wrap_rdbms_result(_) -> ok. - diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index 55d2967f4a..d238ba0475 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -35,10 +35,6 @@ limit_offset_sql/0, limit_offset_args/2, sql_transaction/2, - get_last/2, - select_last/3, - set_last_t/4, - del_last/2, get_password/2, set_password_t/3, add_user/3, @@ -247,30 +243,6 @@ begin_trans(mssql) -> begin_trans(_) -> [<<"BEGIN;">>]. - -get_last(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select seconds, state from last " - "where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -select_last(LServer, TStamp, Comparator) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select username, seconds, state from last " - "where seconds ">>, Comparator, " ", - mongoose_rdbms:use_escaped_integer(mongoose_rdbms:escape_integer(TStamp)), ";"]). - -set_last_t(LServer, Username, Seconds, State) -> - update(LServer, "last", ["username", "seconds", "state"], - [Username, Seconds, State], - [<<"username=">>, mongoose_rdbms:use_escaped_string(Username)]). - -del_last(LServer, Username) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from last where username=">>, mongoose_rdbms:use_escaped_string(Username)]). - get_password(LServer, Username) -> mongoose_rdbms:sql_query( LServer, From 5372539526aa2b74b27764cce872eb09f64dd3b9 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 4 Feb 2021 13:20:17 +0100 Subject: [PATCH 86/92] Remove unused rdbms_queries:update/5 function --- src/rdbms/rdbms_queries.erl | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index d238ba0475..e5ce2df142 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -113,24 +113,6 @@ update_t(Table, Fields, Vals, Where) -> join_escaped(Vals) -> join([mongoose_rdbms:use_escaped(X) || X <- Vals], ", "). - -update(LServer, Table, Fields, Vals, Where) -> - UPairs = lists:zipwith(fun(A, B) -> [A, "=", mongoose_rdbms:use_escaped(B)] end, - Fields, Vals), - case mongoose_rdbms:sql_query( - LServer, - [<<"update ">>, Table, <<" set ">>, - join(UPairs, ", "), - <<" where ">>, Where, ";"]) of - {updated, 1} -> - ok; - _ -> - mongoose_rdbms:sql_query( - LServer, - [<<"insert into ">>, Table, "(", join(Fields, ", "), - <<") values (">>, join_escaped(Vals), ");"]) - end. - -spec execute_upsert(Host :: mongoose_rdbms:server(), Name :: atom(), InsertParams :: [any()], From c3ff8c8e5d03fa60b59c4deb17ea822d8e2ffc80 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 18 Feb 2021 08:40:52 +0100 Subject: [PATCH 87/92] Insert only a diff of privacy_data Retry privacy update transaction Update eodbc. Return bigints as numbers, not binaries now. Smaller data type for the ord field. --- priv/azuresql.sql | 2 +- priv/mssql2012.sql | 2 +- priv/mysql.sql | 2 +- priv/pg.sql | 2 +- rebar.lock | 2 +- src/mod_privacy_rdbms.erl | 71 +++++++++++++++++++++++++++++---------- 6 files changed, 59 insertions(+), 22 deletions(-) diff --git a/priv/azuresql.sql b/priv/azuresql.sql index a07dea0d07..6d33f4b448 100644 --- a/priv/azuresql.sql +++ b/priv/azuresql.sql @@ -262,7 +262,7 @@ CREATE TABLE [dbo].[privacy_list_data]( [t] [char](1) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL, [value] [varchar](max) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL, [action] [char](1) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL, - [ord] [bigint] NOT NULL, + [ord] [int] NOT NULL, [match_all] [smallint] NOT NULL, [match_iq] [smallint] NOT NULL, [match_message] [smallint] NOT NULL, diff --git a/priv/mssql2012.sql b/priv/mssql2012.sql index 288c92582e..7818ab19a0 100644 --- a/priv/mssql2012.sql +++ b/priv/mssql2012.sql @@ -220,7 +220,7 @@ CREATE TABLE [dbo].[privacy_list_data]( [t] [char](1) NOT NULL, [value] [nvarchar](max) NOT NULL, [action] [char](1) NOT NULL, - [ord] [bigint] NOT NULL, + [ord] [int] NOT NULL, [match_all] [smallint] NOT NULL, [match_iq] [smallint] NOT NULL, [match_message] [smallint] NOT NULL, diff --git a/priv/mysql.sql b/priv/mysql.sql index 046ba176f7..172eb11331 100644 --- a/priv/mysql.sql +++ b/priv/mysql.sql @@ -157,7 +157,7 @@ CREATE TABLE privacy_list_data ( t character(1) NOT NULL, value text NOT NULL, action character(1) NOT NULL, - ord bigint NOT NULL, + ord INT NOT NULL, match_all boolean NOT NULL, match_iq boolean NOT NULL, match_message boolean NOT NULL, diff --git a/priv/pg.sql b/priv/pg.sql index 368f91644f..eed593179b 100644 --- a/priv/pg.sql +++ b/priv/pg.sql @@ -143,7 +143,7 @@ CREATE TABLE privacy_list_data ( t character(1) NOT NULL, value text NOT NULL, action character(1) NOT NULL, - ord NUMERIC NOT NULL, + ord INT NOT NULL, match_all boolean NOT NULL, match_iq boolean NOT NULL, match_message boolean NOT NULL, diff --git a/rebar.lock b/rebar.lock index ec362aafe7..a19ec19f0e 100644 --- a/rebar.lock +++ b/rebar.lock @@ -26,7 +26,7 @@ {<<"eini">>,{pkg,<<"eini">>,<<"1.2.7">>},1}, {<<"eodbc">>, {git,"https://github.com/arcusfelis/eodbc.git", - {ref,"612461e4d8e12c0947a67e0bfdeb854010db0b2d"}}, + {ref,"1823d8fe6f5fbe2d8724a9649b75ebd5b8738661"}}, 0}, {<<"eper">>, {git,"http://github.com/basho/eper.git", diff --git a/src/mod_privacy_rdbms.erl b/src/mod_privacy_rdbms.erl index 1b2f8c0932..b4399319be 100644 --- a/src/mod_privacy_rdbms.erl +++ b/src/mod_privacy_rdbms.erl @@ -68,17 +68,26 @@ prepare_queries(Host) -> prepare_default_list_upsert(Host), %% Queries to privacy_list_data table mongoose_rdbms:prepare(privacy_data_get_by_id, privacy_list_data, [id], - <<"SELECT t, value, action, ord, match_all, match_iq, " + <<"SELECT ord, t, value, action, match_all, match_iq, " "match_message, match_presence_in, match_presence_out " "FROM privacy_list_data " "WHERE id=? ORDER BY ord">>), mongoose_rdbms:prepare(delete_data_by_id, privacy_list_data, [id], <<"DELETE FROM privacy_list_data WHERE id=?">>), + mongoose_rdbms:prepare(privacy_data_delete, privacy_list_data, [id, ord], + <<"DELETE FROM privacy_list_data WHERE id=? AND ord=?">>), + mongoose_rdbms:prepare(privacy_data_update, privacy_list_data, + [t, value, action, match_all, match_iq, + match_message, match_presence_in, match_presence_out, id, ord], + <<"UPDATE privacy_list_data SET " + "t=?, value=?, action=?, match_all=?, match_iq=?, " + "match_message=?, match_presence_in=?, match_presence_out=? " + " WHERE id=? AND ord=?">>), mongoose_rdbms:prepare(privacy_data_insert, privacy_list_data, - [id, t, value, action, ord, match_all, match_iq, + [id, ord, t, value, action, match_all, match_iq, match_message, match_presence_in, match_presence_out], <<"INSERT INTO privacy_list_data(" - "id, t, value, action, ord, match_all, match_iq, " + "id, ord, t, value, action, match_all, match_iq, " "match_message, match_presence_in, match_presence_out) " "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)">>), %% This query uses multiple tables @@ -269,14 +278,8 @@ replace_privacy_list(LUser, LServer, Name, List) -> replace_data_rows(LServer, ID, Rows), ok end, - case rdbms_queries:sql_transaction(LServer, F) of - {atomic, ok} -> - ok; - {aborted, Reason} -> - {error, {aborted, Reason}}; - {error, Reason} -> - {error, Reason} - end. + {atomic, ok} = mongoose_rdbms:transaction_with_delayed_retry(LServer, F, #{retries => 5, delay => 100}), + ok. remove_user(LUser, LServer) -> F = fun() -> remove_user_t(LUser, LServer) end, @@ -311,18 +314,52 @@ execute_privacy_list_insert(LServer, LUser, Name) -> execute_delete_data_by_id(LServer, ID) -> mongoose_rdbms:execute_successfully(LServer, delete_data_by_id, [ID]). -replace_data_rows(LServer, ID, Rows) when is_integer(ID)-> - execute_delete_data_by_id(LServer, ID), - [mongoose_rdbms:execute_successfully(LServer, privacy_data_insert, [ID|Args]) - || Args <- Rows], +replace_data_rows(LServer, ID, []) when is_integer(ID) -> + %% Just remove the data, nothing should be inserted + execute_delete_data_by_id(LServer, ID); +replace_data_rows(LServer, ID, Rows) when is_integer(ID) -> + {selected, OldRows} = execute_privacy_data_get_by_id(LServer, ID), + New = lists:sort(Rows), + Old = lists:sort([tuple_to_list(Row) || Row <- OldRows]), + Diff = diff_rows(ID, New, Old, []), + F = fun({Q, Args}) -> mongoose_rdbms:execute_successfully(LServer, Q, Args) end, + lists:foreach(F, Diff), ok. +%% We assume that there are no record duplicates with the same Order. +%% It's checked in the main module for the New argument. +%% It's checked by the database for the Old argument. +diff_rows(ID, [H|New], [H|Old], Ops) -> + diff_rows(ID, New, Old, Ops); %% Not modified +diff_rows(ID, [NewH|NewT] = New, [OldH|OldT] = Old, Ops) -> + NewOrder = hd(NewH), + OldOrder = hd(OldH), + if NewOrder =:= OldOrder -> + Op = {privacy_data_update, tl(NewH) ++ [ID, OldOrder]}, + diff_rows(ID, NewT, OldT, [Op|Ops]); + NewOrder > OldOrder -> + Op = {privacy_data_delete, [ID, OldOrder]}, + diff_rows(ID, New, OldT, [Op|Ops]); + true -> + Op = {privacy_data_insert, [ID|NewH]}, + diff_rows(ID, NewT, Old, [Op|Ops]) + end; +diff_rows(ID, [], [OldH|OldT], Ops) -> + OldOrder = hd(OldH), + Op = {privacy_data_delete, [ID, OldOrder]}, + diff_rows(ID, [], OldT, [Op|Ops]); +diff_rows(ID, [NewH|NewT], [], Ops) -> + Op = {privacy_data_insert, [ID|NewH]}, + diff_rows(ID, NewT, [], [Op|Ops]); +diff_rows(_ID, [], [], Ops) -> + Ops. + %% Encoding/decoding pure functions raw_to_items(Rows) -> [raw_to_item(Row) || Row <- Rows]. -raw_to_item({ExtType, ExtValue, ExtAction, ExtOrder, +raw_to_item({ExtOrder, ExtType, ExtValue, ExtAction, ExtMatchAll, ExtMatchIQ, ExtMatchMessage, ExtMatchPresenceIn, ExtMatchPresenceOut}) -> Type = decode_type(mongoose_rdbms:character_to_integer(ExtType)), @@ -352,7 +389,7 @@ item_to_raw(#listitem{type = Type, ExtAction = encode_action(Action), Bools = [MatchAll, MatchIQ, MatchMessage, MatchPresenceIn, MatchPresenceOut], ExtBools = [encode_boolean(X) || X <- Bools], - [ExtType, ExtValue, ExtAction, Order | ExtBools]. + [Order, ExtType, ExtValue, ExtAction | ExtBools]. encode_boolean(true) -> 1; encode_boolean(false) -> 0. From d7137ed5637ff3ef888f04c427c04e37adaf1ca6 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 4 Feb 2021 13:16:26 +0100 Subject: [PATCH 88/92] Use prepared queries for mod_offline --- src/offline/mod_offline_rdbms.erl | 240 +++++++++++++++++------------- src/rdbms/rdbms_queries.erl | 99 +----------- src/rdbms/rdbms_queries_mssql.erl | 20 +-- 3 files changed, 141 insertions(+), 218 deletions(-) diff --git a/src/offline/mod_offline_rdbms.erl b/src/offline/mod_offline_rdbms.erl index 9a3019ceb5..d5997f4abe 100644 --- a/src/offline/mod_offline_rdbms.erl +++ b/src/offline/mod_offline_rdbms.erl @@ -34,6 +34,8 @@ remove_old_messages/2, remove_user/2]). +-import(mongoose_rdbms, [prepare/4, execute_successfully/3]). + -include("mongoose.hrl"). -include("jlib.hrl"). -include("mod_offline.hrl"). @@ -41,15 +43,82 @@ -define(OFFLINE_TABLE_LOCK_THRESHOLD, 1000). init(_Host, _Opts) -> + prepare_queries(), + ok. + +prepare_queries() -> + prepare(offline_insert, offline_message, + [username, server, from_jid, timestamp, expire, + packet, permanent_fields], + <<"INSERT INTO offline_message " + "(username, server, from_jid, timestamp, expire," + " packet, permanent_fields) " + "VALUES (?, ?, ?, ?, ?, ?, ?)">>), + {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(), + prepare(offline_count_limit, offline_message, + rdbms_queries:add_limit_arg(limit, [server, username]), + <<"SELECT ", LimitMSSQL/binary, + " count(*) FROM offline_message " + "WHERE server=? AND username=? ", LimitSQL/binary>>), + prepare(offline_select, offline_message, + [server, username, expire], + <<"SELECT timestamp, from_jid, packet, permanent_fields " + "FROM offline_message " + "WHERE server = ? AND username = ? AND " + "(expire IS null OR expire > ?) " + "ORDER BY timestamp">>), + prepare(offline_delete, offline_message, + [server, username], + <<"DELETE FROM offline_message " + "WHERE server = ? AND username = ?">>), + prepare(offline_delete_old, offline_message, + [timestamp], + <<"DELETE FROM offline_message WHERE timestamp < ?">>), + prepare(offline_delete_expired, offline_message, + [expire], + <<"DELETE FROM offline_message " + "WHERE expire IS NOT null AND expire < ?">>), ok. +execute_count_offline_messages(LUser, LServer, Limit) -> + Args = rdbms_queries:add_limit_arg(Limit, [LServer, LUser]), + execute_successfully(LServer, offline_count_limit, Args). + +execute_fetch_offline_messages(LServer, LUser, ExtTimeStamp) -> + execute_successfully(LServer, offline_select, [LServer, LUser, ExtTimeStamp]). + +execute_remove_expired_offline_messages(LServer, ExtTimeStamp) -> + execute_successfully(LServer, offline_delete_expired, [ExtTimeStamp]). + +execute_remove_old_offline_messages(LServer, ExtTimeStamp) -> + execute_successfully(LServer, offline_delete_old, [ExtTimeStamp]). + +execute_remove_user(LServer, LUser) -> + execute_successfully(LServer, offline_delete, [LServer, LUser]). + +%% Transactions + +pop_offline_messages(LServer, LUser, ExtTimeStamp) -> + F = fun() -> + Res = execute_fetch_offline_messages(LServer, LUser, ExtTimeStamp), + execute_remove_user(LServer, LUser), + Res + end, + mongoose_rdbms:sql_transaction(LServer, F). + +push_offline_messages(LServer, Rows) -> + F = fun() -> + [execute_successfully(LServer, offline_insert, Row) + || Row <- Rows], ok + end, + mongoose_rdbms:sql_transaction(LServer, F). + +%% API functions + pop_messages(#jid{} = To) -> US = {LUser, LServer} = jid:to_lus(To), - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - TimeStamp = erlang:system_time(microsecond), - STimeStamp = encode_timestamp(TimeStamp), - case rdbms_queries:pop_offline_messages(LServer, SUser, SServer, STimeStamp) of + ExtTimeStamp = os:system_time(microsecond), + case pop_offline_messages(LServer, LUser, ExtTimeStamp) of {atomic, {selected, Rows}} -> {ok, rows_to_records(US, To, Rows)}; {aborted, Reason} -> @@ -58,123 +127,84 @@ pop_messages(#jid{} = To) -> {error, Reason} end. +%% Fetch messages for GDPR +%% User and Server are not normalized fetch_messages(#jid{} = To) -> US = {LUser, LServer} = jid:to_lus(To), - TimeStamp = erlang:system_time(microsecond), - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - STimeStamp = encode_timestamp(TimeStamp), - case rdbms_queries:fetch_offline_messages(LServer, SUser, SServer, STimeStamp) of - {selected, Rows} -> - {ok, rows_to_records(US, To, Rows)}; - {error, Reason} -> - {error, Reason} - end. - -rows_to_records(US, To, Rows) -> - [row_to_record(US, To, Row) || Row <- Rows]. - -row_to_record(US, To, {STimeStamp, SFrom, SPacket, SPermanentFields}) -> - {ok, Packet} = exml:parse(SPacket), - TimeStamp = mongoose_rdbms:result_to_integer(STimeStamp), - From = jid:from_binary(SFrom), - PermanentFields = extract_permanent_fields(SPermanentFields), - #offline_msg{us = US, - timestamp = TimeStamp, - expire = never, - from = From, - to = To, - packet = Packet, - permanent_fields = PermanentFields}. - -extract_permanent_fields(null) -> - []; %% This is needed in transition period when upgrading to MongooseIM above 3.5.0 -extract_permanent_fields(Term) -> - Unescaped = mongoose_rdbms:unescape_binary(global, Term), - binary_to_term(Unescaped). + ExtTimeStamp = os:system_time(microsecond), + {selected, Rows} = execute_fetch_offline_messages(LServer, LUser, ExtTimeStamp), + {ok, rows_to_records(US, To, Rows)}. write_messages(LUser, LServer, Msgs) -> - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - write_all_messages_t(LServer, SUser, SServer, Msgs). - -count_offline_messages(LUser, LServer, MaxArchivedMsgs) -> - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - count_offline_messages(LUser, LServer, SUser, SServer, MaxArchivedMsgs + 1). - -write_all_messages_t(LServer, SUser, SServer, Msgs) -> - Rows = [record_to_row(SUser, SServer, Msg) || Msg <- Msgs], - case rdbms_queries:push_offline_messages(LServer, Rows) of - {updated, _} -> + Rows = [record_to_row(LUser, LServer, Msg) || Msg <- Msgs], + case push_offline_messages(LServer, Rows) of + {atomic, ok} -> ok; - {aborted, Reason} -> - {error, {aborted, Reason}}; - {error, Reason} -> - {error, Reason} + Other -> + {error, Other} end. -record_to_row(SUser, SServer, - #offline_msg{from = From, packet = Packet, timestamp = TimeStamp, - expire = Expire, permanent_fields = PermanentFields}) -> - SFrom = mongoose_rdbms:escape_string(jid:to_binary(From)), - SPacket = mongoose_rdbms:escape_string(exml:to_binary(Packet)), - STimeStamp = encode_timestamp(TimeStamp), - SExpire = maybe_encode_timestamp(Expire), - SFields = encode_permanent_fields(PermanentFields), - rdbms_queries:prepare_offline_message(SUser, SServer, STimeStamp, SExpire, SFrom, SPacket, SFields). - -encode_permanent_fields(Fields) -> - Binary = term_to_binary(Fields), - mongoose_rdbms:escape_binary(global, Binary). +count_offline_messages(LUser, LServer, MaxArchivedMsgs) -> + Result = execute_count_offline_messages(LUser, LServer, MaxArchivedMsgs + 1), + mongoose_rdbms:selected_to_integer(Result). remove_user(LUser, LServer) -> - SUser = mongoose_rdbms:escape_string(LUser), - SServer = mongoose_rdbms:escape_string(LServer), - rdbms_queries:remove_offline_messages(LServer, SUser, SServer). + execute_remove_user(LServer, LUser). --spec remove_expired_messages(jid:lserver()) -> {error, term()} | {ok, HowManyRemoved} when - HowManyRemoved :: integer(). +-spec remove_expired_messages(jid:lserver()) -> + {error, term()} | {ok, HowManyRemoved :: non_neg_integer()}. remove_expired_messages(LServer) -> - TimeStamp = erlang:system_time(microsecond), - STimeStamp = encode_timestamp(TimeStamp), - Result = rdbms_queries:remove_expired_offline_messages(LServer, STimeStamp), - case Result of - {error, Reason} -> - {error, Reason}; - {updated, Count} -> - {ok, Count} - end. + ExtTimeStamp = os:system_time(microsecond), + Result = execute_remove_expired_offline_messages(LServer, ExtTimeStamp), + updated_ok(Result). + -spec remove_old_messages(LServer, Timestamp) -> {error, term()} | {ok, HowManyRemoved} when LServer :: jid:lserver(), Timestamp :: integer(), HowManyRemoved :: integer(). remove_old_messages(LServer, TimeStamp) -> - STimeStamp = encode_timestamp(TimeStamp), - Result = rdbms_queries:remove_old_offline_messages(LServer, STimeStamp), - case Result of - {error, Reason} -> - {error, Reason}; - {updated, Count} -> - {ok, Count} - end. + ExtTimeStamp = encode_timestamp(TimeStamp), + Result = execute_remove_old_offline_messages(LServer, ExtTimeStamp), + updated_ok(Result). -count_offline_messages(LUser, LServer, SUser, SServer, Limit) -> - case rdbms_queries:count_offline_messages(LServer, SUser, SServer, Limit) of - {selected, [{Count}]} -> - mongoose_rdbms:result_to_integer(Count); - Error -> - ?LOG_ERROR(#{what => count_offline_messages_failed, - server => LServer, user => LUser, - reason => Error}), - 0 - end. +%% Pure helper functions +record_to_row(LUser, LServer, + #offline_msg{from = From, packet = Packet, timestamp = TimeStamp, + expire = Expire, permanent_fields = PermanentFields}) -> + ExtFrom = jid:to_binary(From), + ExtTimeStamp = encode_timestamp(TimeStamp), + ExtExpire = maybe_encode_timestamp(Expire), + ExtPacket = exml:to_binary(Packet), + ExtFields = encode_permanent_fields(PermanentFields), + prepare_offline_message(LUser, LServer, ExtFrom, ExtTimeStamp,ExtExpire, + ExtPacket, ExtFields). + +prepare_offline_message(LUser, LServer, ExtFrom, ExtTimeStamp, ExtExpire, ExtPacket, ExtFields) -> + [LUser, LServer, ExtFrom, ExtTimeStamp, ExtExpire, ExtPacket, ExtFields]. + +encode_permanent_fields(Fields) -> + term_to_binary(Fields). + +encode_timestamp(TimeStamp) -> usec:from_now(TimeStamp). %% to microseconds + +maybe_encode_timestamp(never) -> null; +maybe_encode_timestamp(TimeStamp) -> encode_timestamp(TimeStamp). + +rows_to_records(US, To, Rows) -> + [row_to_record(US, To, Row) || Row <- Rows]. + +row_to_record(US, To, {ExtTimeStamp, ExtFrom, ExtPacket, ExtPermanentFields}) -> + {ok, Packet} = exml:parse(ExtPacket), + TimeStamp = usec:to_now(mongoose_rdbms:result_to_integer(ExtTimeStamp)), + From = jid:from_binary(ExtFrom), + PermanentFields = extract_permanent_fields(ExtPermanentFields), + #offline_msg{us = US, timestamp = TimeStamp, expire = never, + from = From, to = To, packet = Packet, + permanent_fields = PermanentFields}. -encode_timestamp(TimeStamp) -> - mongoose_rdbms:escape_integer(TimeStamp). +extract_permanent_fields(Escaped) -> + Bin = mongoose_rdbms:unescape_binary(global, Escaped), + binary_to_term(Bin). -maybe_encode_timestamp(never) -> - mongoose_rdbms:escape_null(); -maybe_encode_timestamp(TimeStamp) -> - encode_timestamp(TimeStamp). +updated_ok({updated, Count}) -> {ok, Count}. diff --git a/src/rdbms/rdbms_queries.erl b/src/rdbms/rdbms_queries.erl index e5ce2df142..ad4edc4f80 100644 --- a/src/rdbms/rdbms_queries.erl +++ b/src/rdbms/rdbms_queries.erl @@ -29,6 +29,7 @@ -export([get_db_type/0, begin_trans/0, get_db_specific_limits/0, + get_db_specific_limits_binaries/0, get_db_specific_limits_binaries/1, get_db_specific_offset/2, add_limit_arg/2, @@ -47,14 +48,6 @@ get_users_without_scram/2, get_users_without_scram_count/1, count_records_where/3, - prepare_offline_message/7, - push_offline_messages/2, - pop_offline_messages/4, - fetch_offline_messages/4, - count_offline_messages/4, - remove_old_offline_messages/2, - remove_expired_offline_messages/2, - remove_offline_messages/3, create_bulk_insert_query/3]). -export([join/2, @@ -361,92 +354,6 @@ count_records_where(LServer, Table, WhereClause) -> LServer, [<<"select count(*) from ">>, Table, " ", WhereClause, ";"]). - -pop_offline_messages(LServer, SUser, SServer, STimeStamp) -> - SelectSQL = select_offline_messages_sql(SUser, SServer, STimeStamp), - DeleteSQL = delete_offline_messages_sql(SUser, SServer), - F = fun() -> - Res = mongoose_rdbms:sql_query_t(SelectSQL), - mongoose_rdbms:sql_query_t(DeleteSQL), - Res - end, - mongoose_rdbms:sql_transaction(LServer, F). - -fetch_offline_messages(LServer, SUser, SServer, STimeStamp) -> - mongoose_rdbms:sql_query(LServer, select_offline_messages_sql(SUser, SServer, STimeStamp)). - -select_offline_messages_sql(SUser, SServer, STimeStamp) -> - [<<"select timestamp, from_jid, packet, permanent_fields from offline_message " - "where server = ">>, mongoose_rdbms:use_escaped_string(SServer), <<" and " - "username = ">>, mongoose_rdbms:use_escaped_string(SUser), <<" and " - "(expire is null or expire > ">>, mongoose_rdbms:use_escaped_integer(STimeStamp), <<") " - "ORDER BY timestamp">>]. - -delete_offline_messages_sql(SUser, SServer) -> - [<<"delete from offline_message " - "where server = ">>, mongoose_rdbms:use_escaped_string(SServer), <<" and " - "username = ">>, mongoose_rdbms:use_escaped_string(SUser)]. - -remove_old_offline_messages(LServer, STimeStamp) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from offline_message where timestamp < ">>, - mongoose_rdbms:use_escaped_integer(STimeStamp)]). - -remove_expired_offline_messages(LServer, STimeStamp) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from offline_message " - "where expire is not null and expire < ">>, - mongoose_rdbms:use_escaped_integer(STimeStamp)]). - -remove_offline_messages(LServer, SUser, SServer) -> - mongoose_rdbms:sql_query( - LServer, - [<<"delete from offline_message " - "where server = ">>, mongoose_rdbms:use_escaped_string(SServer), <<" and " - "username = ">>, mongoose_rdbms:use_escaped_string(SUser)]). - --spec prepare_offline_message(SUser, SServer, STimeStamp, SExpire, SFrom, SPacket, SFields) -> - mongoose_rdbms:sql_query_part() when - SUser :: mongoose_rdbms:escaped_string(), - SServer :: mongoose_rdbms:escaped_string(), - STimeStamp :: mongoose_rdbms:escaped_timestamp(), - SExpire :: mongoose_rdbms:escaped_timestamp() | mongoose_rdbms:escaped_null(), - SFrom :: mongoose_rdbms:escaped_string(), - SPacket :: mongoose_rdbms:escaped_string(), - SFields :: mongoose_rdbms:escaped_binary(). -prepare_offline_message(SUser, SServer, STimeStamp, SExpire, SFrom, SPacket, SFields) -> - [<<"(">>, mongoose_rdbms:use_escaped_string(SUser), - <<", ">>, mongoose_rdbms:use_escaped_string(SServer), - <<", ">>, mongoose_rdbms:use_escaped_integer(STimeStamp), - <<", ">>, mongoose_rdbms:use_escaped(SExpire), - <<", ">>, mongoose_rdbms:use_escaped_string(SFrom), - <<", ">>, mongoose_rdbms:use_escaped_string(SPacket), - <<", ">>, mongoose_rdbms:use_escaped_binary(SFields), - <<")">>]. - -push_offline_messages(LServer, Rows) -> - mongoose_rdbms:sql_query( - LServer, - [<<"INSERT INTO offline_message " - "(username, server, timestamp, expire, from_jid, packet, permanent_fields) " - "VALUES ">>, join(Rows, ", ")]). - - -count_offline_messages(LServer, SUser, SServer, Limit) -> - count_offline_messages(?RDBMS_TYPE, LServer, SUser, SServer, Limit). - -count_offline_messages(mssql, LServer, SUser, SServer, Limit) -> - rdbms_queries_mssql:count_offline_messages(LServer, SUser, SServer, Limit); -count_offline_messages(_, LServer, SUser, SServer, Limit) -> - mongoose_rdbms:sql_query( - LServer, - [<<"select count(*) from offline_message " - "where server = ">>, mongoose_rdbms:use_escaped_string(SServer), <<" and " - "username = ">>, mongoose_rdbms:use_escaped_string(SUser), <<" " - "limit ">>, integer_to_list(Limit)]). - -spec create_bulk_insert_query(Table :: iodata() | atom(), Fields :: [iodata() | atom()], RowsNum :: pos_integer()) -> {iodata(), [binary()]}. @@ -468,6 +375,10 @@ create_bulk_insert_query(Table, Fields, RowsNum) when RowsNum > 0 -> get_db_specific_limits() -> do_get_db_specific_limits(?RDBMS_TYPE, "?", true). +get_db_specific_limits_binaries() -> + {LimitSQL, LimitMSSQL} = get_db_specific_limits(), + {list_to_binary(LimitSQL), list_to_binary(LimitMSSQL)}. + -spec get_db_specific_limits(integer()) -> {SQL :: nonempty_string(), []} | {[], MSSQL::nonempty_string()}. get_db_specific_limits(Limit) -> diff --git a/src/rdbms/rdbms_queries_mssql.erl b/src/rdbms/rdbms_queries_mssql.erl index 0f8eba352c..64b05ea251 100644 --- a/src/rdbms/rdbms_queries_mssql.erl +++ b/src/rdbms/rdbms_queries_mssql.erl @@ -26,26 +26,8 @@ -include("mongoose.hrl"). %% API --export([begin_trans/0, - query_archive_id/3, - count_offline_messages/4]). +-export([begin_trans/0]). begin_trans() -> [<<"BEGIN TRANSACTION;">>]. - -query_archive_id(Host, SServer, SUserName) -> - mongoose_rdbms:sql_query( - Host, - ["SELECT TOP 1 id " - "FROM mam_server_user " - "WHERE server=", mongoose_rdbms:use_escaped_string(SServer), - " AND user_name=", mongoose_rdbms:use_escaped_string(SUserName)]). - -count_offline_messages(LServer, SUser, SServer, Limit) -> - mongoose_rdbms:sql_query( - LServer, - [<<"SELECT TOP ">>, integer_to_list(Limit), - <<"count(*) FROM offline_message " - "WHERE server=">>, mongoose_rdbms:use_escaped_string(SServer), - <<" AND username=">>, mongoose_rdbms:use_escaped_string(SUser)]). From a455d25bc49c0969247e979c8879bde54a055d3e Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 23 Feb 2021 09:32:48 +0100 Subject: [PATCH 89/92] Address review comments --- src/offline/mod_offline_rdbms.erl | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/offline/mod_offline_rdbms.erl b/src/offline/mod_offline_rdbms.erl index d5997f4abe..5c07ac9e87 100644 --- a/src/offline/mod_offline_rdbms.erl +++ b/src/offline/mod_offline_rdbms.erl @@ -51,8 +51,8 @@ prepare_queries() -> [username, server, from_jid, timestamp, expire, packet, permanent_fields], <<"INSERT INTO offline_message " - "(username, server, from_jid, timestamp, expire," - " packet, permanent_fields) " + "(username, server, timestamp, expire," + " from_jid, packet, permanent_fields) " "VALUES (?, ?, ?, ?, ?, ?, ?)">>), {LimitSQL, LimitMSSQL} = rdbms_queries:get_db_specific_limits_binaries(), prepare(offline_count_limit, offline_message, @@ -93,7 +93,7 @@ execute_remove_expired_offline_messages(LServer, ExtTimeStamp) -> execute_remove_old_offline_messages(LServer, ExtTimeStamp) -> execute_successfully(LServer, offline_delete_old, [ExtTimeStamp]). -execute_remove_user(LServer, LUser) -> +execute_offline_delete(LServer, LUser) -> execute_successfully(LServer, offline_delete, [LServer, LUser]). %% Transactions @@ -101,7 +101,7 @@ execute_remove_user(LServer, LUser) -> pop_offline_messages(LServer, LUser, ExtTimeStamp) -> F = fun() -> Res = execute_fetch_offline_messages(LServer, LUser, ExtTimeStamp), - execute_remove_user(LServer, LUser), + execute_offline_delete(LServer, LUser), Res end, mongoose_rdbms:sql_transaction(LServer, F). @@ -149,7 +149,7 @@ count_offline_messages(LUser, LServer, MaxArchivedMsgs) -> mongoose_rdbms:selected_to_integer(Result). remove_user(LUser, LServer) -> - execute_remove_user(LServer, LUser). + execute_offline_delete(LServer, LUser). -spec remove_expired_messages(jid:lserver()) -> {error, term()} | {ok, HowManyRemoved :: non_neg_integer()}. @@ -170,18 +170,18 @@ remove_old_messages(LServer, TimeStamp) -> %% Pure helper functions record_to_row(LUser, LServer, - #offline_msg{from = From, packet = Packet, timestamp = TimeStamp, - expire = Expire, permanent_fields = PermanentFields}) -> - ExtFrom = jid:to_binary(From), + #offline_msg{timestamp = TimeStamp, expire = Expire, from = From, + packet = Packet, permanent_fields = PermanentFields}) -> ExtTimeStamp = encode_timestamp(TimeStamp), ExtExpire = maybe_encode_timestamp(Expire), + ExtFrom = jid:to_binary(From), ExtPacket = exml:to_binary(Packet), ExtFields = encode_permanent_fields(PermanentFields), - prepare_offline_message(LUser, LServer, ExtFrom, ExtTimeStamp,ExtExpire, - ExtPacket, ExtFields). + prepare_offline_message(LUser, LServer, ExtTimeStamp, ExtExpire, + ExtFrom, ExtPacket, ExtFields). -prepare_offline_message(LUser, LServer, ExtFrom, ExtTimeStamp, ExtExpire, ExtPacket, ExtFields) -> - [LUser, LServer, ExtFrom, ExtTimeStamp, ExtExpire, ExtPacket, ExtFields]. +prepare_offline_message(LUser, LServer, ExtTimeStamp, ExtExpire, ExtFrom, ExtPacket, ExtFields) -> + [LUser, LServer, ExtTimeStamp, ExtExpire, ExtFrom, ExtPacket, ExtFields]. encode_permanent_fields(Fields) -> term_to_binary(Fields). @@ -203,6 +203,8 @@ row_to_record(US, To, {ExtTimeStamp, ExtFrom, ExtPacket, ExtPermanentFields}) -> from = From, to = To, packet = Packet, permanent_fields = PermanentFields}. +extract_permanent_fields(null) -> + []; %% This is needed in transition period when upgrading to MongooseIM above 3.5.0 extract_permanent_fields(Escaped) -> Bin = mongoose_rdbms:unescape_binary(global, Escaped), binary_to_term(Bin). From fee25ca4d107963b60e83cd5cf70f538e7f98909 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 25 Feb 2021 13:58:05 +0100 Subject: [PATCH 90/92] Don't use now format for dates --- src/offline/mod_offline_rdbms.erl | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/offline/mod_offline_rdbms.erl b/src/offline/mod_offline_rdbms.erl index 5c07ac9e87..8b65885c5f 100644 --- a/src/offline/mod_offline_rdbms.erl +++ b/src/offline/mod_offline_rdbms.erl @@ -154,8 +154,8 @@ remove_user(LUser, LServer) -> -spec remove_expired_messages(jid:lserver()) -> {error, term()} | {ok, HowManyRemoved :: non_neg_integer()}. remove_expired_messages(LServer) -> - ExtTimeStamp = os:system_time(microsecond), - Result = execute_remove_expired_offline_messages(LServer, ExtTimeStamp), + TimeStamp = os:system_time(microsecond), + Result = execute_remove_expired_offline_messages(LServer, TimeStamp), updated_ok(Result). -spec remove_old_messages(LServer, Timestamp) -> @@ -164,20 +164,18 @@ remove_expired_messages(LServer) -> Timestamp :: integer(), HowManyRemoved :: integer(). remove_old_messages(LServer, TimeStamp) -> - ExtTimeStamp = encode_timestamp(TimeStamp), - Result = execute_remove_old_offline_messages(LServer, ExtTimeStamp), + Result = execute_remove_old_offline_messages(LServer, TimeStamp), updated_ok(Result). %% Pure helper functions record_to_row(LUser, LServer, #offline_msg{timestamp = TimeStamp, expire = Expire, from = From, packet = Packet, permanent_fields = PermanentFields}) -> - ExtTimeStamp = encode_timestamp(TimeStamp), ExtExpire = maybe_encode_timestamp(Expire), ExtFrom = jid:to_binary(From), ExtPacket = exml:to_binary(Packet), ExtFields = encode_permanent_fields(PermanentFields), - prepare_offline_message(LUser, LServer, ExtTimeStamp, ExtExpire, + prepare_offline_message(LUser, LServer, TimeStamp, ExtExpire, ExtFrom, ExtPacket, ExtFields). prepare_offline_message(LUser, LServer, ExtTimeStamp, ExtExpire, ExtFrom, ExtPacket, ExtFields) -> @@ -186,17 +184,15 @@ prepare_offline_message(LUser, LServer, ExtTimeStamp, ExtExpire, ExtFrom, ExtPac encode_permanent_fields(Fields) -> term_to_binary(Fields). -encode_timestamp(TimeStamp) -> usec:from_now(TimeStamp). %% to microseconds - maybe_encode_timestamp(never) -> null; -maybe_encode_timestamp(TimeStamp) -> encode_timestamp(TimeStamp). +maybe_encode_timestamp(TimeStamp) -> TimeStamp. rows_to_records(US, To, Rows) -> [row_to_record(US, To, Row) || Row <- Rows]. row_to_record(US, To, {ExtTimeStamp, ExtFrom, ExtPacket, ExtPermanentFields}) -> {ok, Packet} = exml:parse(ExtPacket), - TimeStamp = usec:to_now(mongoose_rdbms:result_to_integer(ExtTimeStamp)), + TimeStamp = mongoose_rdbms:result_to_integer(ExtTimeStamp), From = jid:from_binary(ExtFrom), PermanentFields = extract_permanent_fields(ExtPermanentFields), #offline_msg{us = US, timestamp = TimeStamp, expire = never, From 10a76658766417cf6adac6dd6fc918466375b47c Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 25 Feb 2021 15:13:05 +0100 Subject: [PATCH 91/92] Fix order of fields in mod_offline_rdbms --- src/offline/mod_offline_rdbms.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/offline/mod_offline_rdbms.erl b/src/offline/mod_offline_rdbms.erl index 8b65885c5f..75bbf11d74 100644 --- a/src/offline/mod_offline_rdbms.erl +++ b/src/offline/mod_offline_rdbms.erl @@ -48,8 +48,8 @@ init(_Host, _Opts) -> prepare_queries() -> prepare(offline_insert, offline_message, - [username, server, from_jid, timestamp, expire, - packet, permanent_fields], + [username, server, timestamp, expire, + from_jid, packet, permanent_fields], <<"INSERT INTO offline_message " "(username, server, timestamp, expire," " from_jid, packet, permanent_fields) " From 64f4d651726523fa7af1b2b99f1ee12929ccba78 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 1 Mar 2021 12:38:56 +0100 Subject: [PATCH 92/92] Address review comments --- src/offline/mod_offline_rdbms.erl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/offline/mod_offline_rdbms.erl b/src/offline/mod_offline_rdbms.erl index 75bbf11d74..5d38a0f587 100644 --- a/src/offline/mod_offline_rdbms.erl +++ b/src/offline/mod_offline_rdbms.erl @@ -59,7 +59,7 @@ prepare_queries() -> rdbms_queries:add_limit_arg(limit, [server, username]), <<"SELECT ", LimitMSSQL/binary, " count(*) FROM offline_message " - "WHERE server=? AND username=? ", LimitSQL/binary>>), + "WHERE server = ? AND username = ? ", LimitSQL/binary>>), prepare(offline_select, offline_message, [server, username, expire], <<"SELECT timestamp, from_jid, packet, permanent_fields " @@ -77,8 +77,7 @@ prepare_queries() -> prepare(offline_delete_expired, offline_message, [expire], <<"DELETE FROM offline_message " - "WHERE expire IS NOT null AND expire < ?">>), - ok. + "WHERE expire IS NOT null AND expire < ?">>). execute_count_offline_messages(LUser, LServer, Limit) -> Args = rdbms_queries:add_limit_arg(Limit, [LServer, LUser]), @@ -128,7 +127,6 @@ pop_messages(#jid{} = To) -> end. %% Fetch messages for GDPR -%% User and Server are not normalized fetch_messages(#jid{} = To) -> US = {LUser, LServer} = jid:to_lus(To), ExtTimeStamp = os:system_time(microsecond),