Merge 872c8f2 into 53935c5

big_tests/tests/mam_SUITE.erl

@@ -616,15 +616,19 @@ init_per_group(muc_rsm04, Config) ->
 init_per_group(Group, ConfigIn) ->
    C = configuration(Group),
    B = basic_group(Group),
-   case init_modules(C, B, ConfigIn) of
+   try init_modules(C, B, ConfigIn) of
         skip ->
             {skip, print_configuration_not_supported(C, B)};
         Config0 ->
             ct:pal("Init per group ~p; configuration ~p; basic group ~p",
                    [Group, C, B]),
             Config1 = do_init_per_group(C, Config0),
             [{basic_group, B}, {configuration, C} | init_state(C, B, Config1)]
-    end.
+   catch Class:Reason:Stacktrace ->
+             ct:pal("Failed to start configuration=~p basic_group=~p",
+                    [C, B]),
+             erlang:raise(Class, Reason, Stacktrace)
+   end.
 
 backup_module_opts(Module) ->
     {{params_backup, Module}, rpc_apply(gen_mod, get_module_opts, [host(), mod_mam_muc])}.
@@ -753,7 +757,11 @@ init_modules(rdbms_mnesia_cache, C, Config) when C =:= muc_all;
     Config;
 init_modules(BackendType, muc_light, Config) ->
     Config1 = init_modules_for_muc_light(BackendType, Config),
-    init_module(host(), mod_mam_rdbms_user, [muc, pm]),
+    case BackendType of
+        cassandra -> ok;
+        elasticsearch -> ok;
+        _ -> init_module(host(), mod_mam_rdbms_user, [muc, pm])
+    end,
     Config1;
 init_modules(rdbms, C, Config) ->
     init_module(host(), mod_mam, addin_mam_options(C, Config)),

include/mongoose_mam.hrl

@@ -0,0 +1,2 @@
+-record(db_mapping, {column :: atom(), param :: atom(), format :: atom()}).
+-record(lookup_field, {op :: atom(), column :: atom(), param :: atom(), required = false :: boolean(), value_maker :: atom()}).

priv/azuresql.sql

@@ -262,7 +262,7 @@ CREATE TABLE [dbo].[privacy_list_data](
 	[t] [char](1) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL,
 	[value] [varchar](max) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL,
 	[action] [char](1) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL,
-	[ord] [bigint] NOT NULL,
+	[ord] [int] NOT NULL,
 	[match_all] [smallint] NOT NULL,
 	[match_iq] [smallint] NOT NULL,
 	[match_message] [smallint] NOT NULL,

priv/mssql2012.sql

@@ -220,7 +220,7 @@ CREATE TABLE [dbo].[privacy_list_data](
 	[t] [char](1) NOT NULL,
 	[value] [nvarchar](max) NOT NULL,
 	[action] [char](1) NOT NULL,
-	[ord] [bigint] NOT NULL,
+	[ord] [int] NOT NULL,
 	[match_all] [smallint] NOT NULL,
 	[match_iq] [smallint] NOT NULL,
 	[match_message] [smallint] NOT NULL,

priv/mysql.sql

@@ -58,7 +58,7 @@ CREATE INDEX i_last_seconds ON last(seconds);
 
 CREATE TABLE rosterusers (
     username varchar(250) NOT NULL,
-    jid varchar(250) NOT NULL,
+    jid varchar(250) NOT NULL, -- must be a parsable jid
     nick text NOT NULL,
     subscription character(1) NOT NULL,
     ask character(1) NOT NULL,
@@ -157,7 +157,7 @@ CREATE TABLE privacy_list_data (
     t character(1) NOT NULL,
     value text NOT NULL,
     action character(1) NOT NULL,
-    ord bigint NOT NULL,
+    ord INT NOT NULL,
     match_all boolean NOT NULL,
     match_iq boolean NOT NULL,
     match_message boolean NOT NULL,

priv/pg.sql

@@ -143,7 +143,7 @@ CREATE TABLE privacy_list_data (
     t character(1) NOT NULL,
     value text NOT NULL,
     action character(1) NOT NULL,
-    ord NUMERIC NOT NULL,
+    ord INT NOT NULL,
     match_all boolean NOT NULL,
     match_iq boolean NOT NULL,
     match_message boolean NOT NULL,

rebar.lock

@@ -26,7 +26,7 @@
  {<<"eini">>,{pkg,<<"eini">>,<<"1.2.7">>},1},
  {<<"eodbc">>,
   {git,"https://github.com/arcusfelis/eodbc.git",
-       {ref,"612461e4d8e12c0947a67e0bfdeb854010db0b2d"}},
+       {ref,"1823d8fe6f5fbe2d8724a9649b75ebd5b8738661"}},
   0},
  {<<"eper">>,
   {git,"http://github.com/basho/eper.git",

src/mam/mam_decoder.erl

@@ -0,0 +1,53 @@
+-module(mam_decoder).
+-export([decode_row/2]).
+-export([decode_muc_row/2]).
+-export([decode_muc_gdpr_row/2]).
+-export([decode_retraction_info/2]).
+
+-type ext_mess_id() :: non_neg_integer() | binary().
+-type env_vars() :: mod_mam_rdbms_arch:env_vars().
+-type db_row() :: {ext_mess_id(), ExtSrcJID :: binary(), ExtData :: binary()}.
+-type db_muc_row() :: {ext_mess_id(), Nick :: binary(), ExtData :: binary()}.
+-type db_muc_gdpr_row() :: {ext_mess_id(), ExtData :: binary()}.
+-type decoded_muc_gdpr_row() :: {ext_mess_id(), exml:element()}.
+-type retraction_info() :: #{packet := exml:element(), message_id := mod_mam:message_id()}.
+
+-spec decode_row(db_row(), env_vars()) -> mod_mam:message_row().
+decode_row({ExtMessID, ExtSrcJID, ExtData}, Env) ->
+    MessID = mongoose_rdbms:result_to_integer(ExtMessID),
+    SrcJID = decode_jid(ExtSrcJID, Env),
+    Packet = decode_packet(ExtData, Env),
+    {MessID, SrcJID, Packet}.
+
+-spec decode_muc_row(db_muc_row(), env_vars()) -> mod_mam:message_row().
+decode_muc_row({ExtMessID, Nick, ExtData}, Env = #{archive_jid := RoomJID}) ->
+    MessID = mongoose_rdbms:result_to_integer(ExtMessID),
+    SrcJID = jid:replace_resource(RoomJID, Nick),
+    Packet = decode_packet(ExtData, Env),
+    {MessID, SrcJID, Packet}.
+
+-spec decode_muc_gdpr_row(db_muc_gdpr_row(), env_vars()) -> decoded_muc_gdpr_row().
+decode_muc_gdpr_row({ExtMessID, ExtData}, Env) ->
+    Packet = decode_packet(ExtData, Env),
+    {ExtMessID, Packet}.
+
+-spec decode_retraction_info(env_vars(), [] | [{mod_mam:message_id(), binary()}]) ->
+    skip | retraction_info().
+decode_retraction_info(_Env, []) -> skip;
+decode_retraction_info(Env, [{ResMessID, Data}]) ->
+    Packet = decode_packet(Data, Env),
+    MessID = mongoose_rdbms:result_to_integer(ResMessID),
+    #{packet => Packet, message_id => MessID}.
+
+-spec decode_jid(binary(), env_vars()) -> jid:jid().
+decode_jid(ExtJID, #{db_jid_codec := Codec, archive_jid := ArcJID}) ->
+    mam_jid:decode(Codec, ArcJID, ExtJID).
+
+-spec decode_packet(binary(), env_vars()) -> exml:element().
+decode_packet(ExtBin, Env = #{db_message_codec := Codec}) ->
+    Bin = unescape_binary(ExtBin, Env),
+    mam_message:decode(Codec, Bin).
+
+-spec unescape_binary(binary(), env_vars()) -> binary().
+unescape_binary(Bin, #{host := Host}) ->
+    mongoose_rdbms:unescape_binary(Host, Bin).

src/mam/mam_encoder.erl

@@ -0,0 +1,89 @@
+-module(mam_encoder).
+-export([encode_message/3]).
+-export([encode_jid/2]).
+-export([encode_direction/1]).
+-export([encode_packet/2]).
+-export([extend_lookup_params/2]).
+
+-include("mongoose.hrl").
+-include("jlib.hrl").
+-include_lib("exml/include/exml.hrl").
+-include("mongoose_mam.hrl").
+
+-type value_type() :: int | maybe_string | direction | bare_jid | jid | jid_resource | xml | search.
+-type env_vars() :: mod_mam_rdbms_arch:env_vars().
+-type db_mapping() :: #db_mapping{}.
+-type encoded_field_value() :: term().
+
+-spec extend_lookup_params(mam_iq:lookup_params(), env_vars()) -> mam_iq:lookup_params().
+extend_lookup_params(#{start_ts := Start, end_ts := End, with_jid := WithJID,
+                       borders := Borders, search_text := SearchText} = Params, Env) ->
+    Params#{norm_search_text => mod_mam_utils:normalize_search_text(SearchText),
+            start_id => make_start_id(Start, Borders),
+            end_id => make_end_id(End, Borders),
+            remote_bare_jid => maybe_encode_bare_jid(WithJID, Env),
+            remote_resource => jid_to_non_empty_resource(WithJID)}.
+
+-spec encode_message(mod_mam:archive_message_params(), env_vars(), list(db_mapping())) ->
+    [encoded_field_value()].
+encode_message(Params, Env, Mappings) ->
+    [encode_value_using_mapping(Params, Env, Mapping) || Mapping <- Mappings].
+
+encode_value_using_mapping(Params, Env, #db_mapping{param = Param, format = Format}) ->
+    Value = maps:get(Param, Params),
+    encode_value(Format, Value, Env).
+
+-spec encode_value(value_type(), term(), env_vars()) -> encoded_field_value().
+encode_value(int, Value, _Env) when is_integer(Value) ->
+    Value;
+encode_value(maybe_string, none, _Env) ->
+    null;
+encode_value(maybe_string, Value, _Env) when is_binary(Value) ->
+    Value;
+encode_value(direction, Value, _Env) ->
+    encode_direction(Value);
+encode_value(bare_jid, Value, Env) ->
+    encode_jid(jid:to_bare(Value), Env);
+encode_value(jid, Value, Env) ->
+    encode_jid(Value, Env);
+encode_value(jid_resource, #jid{lresource = Res}, _Env) ->
+    Res;
+encode_value(xml, Value, Env) ->
+    encode_packet(Value, Env);
+encode_value(search, Value, Env) ->
+    encode_search_body(Value, Env).
+
+encode_direction(incoming) -> <<"I">>;
+encode_direction(outgoing) -> <<"O">>.
+
+make_start_id(Start, Borders) ->
+    StartID = maybe_encode_compact_uuid(Start, 0),
+    mod_mam_utils:apply_start_border(Borders, StartID).
+
+make_end_id(End, Borders) ->
+    EndID = maybe_encode_compact_uuid(End, 255),
+    mod_mam_utils:apply_end_border(Borders, EndID).
+
+maybe_encode_compact_uuid(undefined, _) ->
+    undefined;
+maybe_encode_compact_uuid(Microseconds, NodeID) ->
+    mod_mam_utils:encode_compact_uuid(Microseconds, NodeID).
+
+jid_to_non_empty_resource(undefined) -> undefined;
+jid_to_non_empty_resource(#jid{lresource = <<>>}) -> undefined;
+jid_to_non_empty_resource(#jid{lresource = Res}) -> Res.
+
+-spec encode_jid(jid:jid(), env_vars()) -> binary().
+encode_jid(JID, #{db_jid_codec := Codec, archive_jid := ArcJID}) ->
+    mam_jid:encode(Codec, ArcJID, JID).
+
+maybe_encode_bare_jid(undefined, _Env) -> undefined;
+maybe_encode_bare_jid(JID, Env) -> encode_jid(jid:to_bare(JID), Env).
+
+-spec encode_packet(exml:element(), env_vars()) -> binary().
+encode_packet(Packet, #{db_message_codec := Codec}) ->
+    mam_message:encode(Codec, Packet).
+
+-spec encode_search_body(exml:element(), env_vars()) -> binary().
+encode_search_body(Packet, #{has_full_text_search := SearchEnabled}) ->
+    mod_mam_utils:packet_to_search_body(SearchEnabled, Packet).

src/mam/mam_filter.erl

@@ -0,0 +1,52 @@
+%% Produces filters based on lookup params
+-module(mam_filter).
+-export([produce_filter/2]).
+-include("mongoose_mam.hrl").
+
+-type column() :: atom().
+
+-type filter_field() :: {like, column(), binary()}
+    | {le, column(), integer()}
+    | {ge, column(), integer()}
+    | {equal, column(), integer() | binary()}
+    | {less, column(), integer()}
+    | {greater, column(), integer()}.
+
+-type filter() :: [filter_field()].
+-type fields() :: [#lookup_field{}].
+-type params() :: map().
+
+-export_type([filter_field/0]).
+-export_type([filter/0]).
+
+-define(SEARCH_WORDS_LIMIT, 10).
+
+-spec produce_filter(params(), fields()) -> list(filter_field()).
+produce_filter(Params, Fields) ->
+    [new_filter(Field, Value)
+     || Field <- Fields,
+        Value <- field_to_values(Field, Params)].
+
+field_to_values(#lookup_field{param = Param, value_maker = ValueMaker, required = Required} = Field, Params) ->
+    case maps:find(Param, Params) of
+        {ok, Value} when Value =/= undefined ->
+            make_value(ValueMaker, Value);
+        Other when Required ->
+            error(#{reason => missing_required_field, field => Field, params => Params, result => Other});
+        _ ->
+            []
+    end.
+
+make_value(search_words, Value) -> search_words(Value);
+make_value(undefined, Value) -> [Value]. %% Default value_maker
+
+new_filter(#lookup_field{op = Op, column = Column}, Value) ->
+    {Op, Column, Value}.
+
+%% Constructs a separate LIKE filter for each word.
+%% SearchText example is "word1%word2%word3".
+%% Order of words does not matter (they can go in any order).
+-spec search_words(binary()) -> list(binary()).
+search_words(SearchText) ->
+    Words = binary:split(SearchText, <<"%">>, [global]),
+    [<<"%", Word/binary, "%">> || Word <- lists:sublist(Words, ?SEARCH_WORDS_LIMIT)].