Unifying c2s metrics

big_tests/tests/bosh_SUITE.erl

@@ -169,6 +169,13 @@ required_bosh_opts(_Group) ->
 %%--------------------------------------------------------------------
 
 create_and_terminate_session(Config) ->
+    MongooseMetrics = [{[global, data, xmpp, received, xml_stanza_size], changed},
+                       {[global, data, xmpp, sent, xml_stanza_size], changed},
+                       {[global, data, xmpp, received, c2s, bosh, raw], changed},
+                       {[global, data, xmpp, sent, c2s, bosh, raw], changed},
+                       {[global, data, xmpp, received, c2s, tcp, raw], 0}, 
+                       {[global, data, xmpp, sent, c2s, tcp, raw], 0}],
+    PreStoryData = escalus_mongooseim:pre_story([{mongoose_metrics, MongooseMetrics}]),
     NamedSpecs = escalus_config:get_config(escalus_users, Config),
     CarolSpec = proplists:get_value(?config(user, Config), NamedSpecs),
     Conn = escalus_connection:connect(CarolSpec),
@@ -188,6 +195,8 @@ create_and_terminate_session(Config) ->
     Terminate = escalus_bosh:session_termination_body(get_bosh_rid(Conn), Sid),
     ok = bosh_send_raw(Conn, Terminate),
 
+    escalus_mongooseim:post_story(PreStoryData),
+
     %% Assert the session was terminated.
     wait_for_zero_bosh_sessions().
 

big_tests/tests/connect_SUITE.erl

@@ -87,6 +87,7 @@ tls_groups()->
         {group, starttls},
         {group, c2s_noproc},
         {group, feature_order},
+        metrics_test, %% must follow right after feature_order group
         {group, tls}
     ].
 
@@ -346,6 +347,18 @@ verify_features(Conn, Features) ->
 get_feature(Feature, FeatureList) ->
     lists:keyfind(Feature, 1, FeatureList).
 
+metrics_test(Config) ->
+    MongooseMetrics = [{[global, data, xmpp, received, xml_stanza_size], changed},
+                       {[global, data, xmpp, sent, xml_stanza_size], changed},
+                       {[global, data, xmpp, received, c2s, tls, raw], changed},
+                       {[global, data, xmpp, sent, c2s, tls, raw], changed},
+                       %% TCP traffic before starttls
+                       {[global, data, xmpp, received, c2s, tcp, raw], changed}, 
+                       {[global, data, xmpp, sent, c2s, tcp, raw], changed}],
+    PreStoryData = escalus_mongooseim:pre_story([{mongoose_metrics, MongooseMetrics}]),
+    tls_authenticate(Config),
+    escalus_mongooseim:post_story(PreStoryData).
+
 tls_authenticate(Config) ->
     %% Given
     UserSpec = escalus_fresh:create_fresh_user(Config, ?SECURE_USER),

big_tests/tests/graphql_metric_SUITE.erl

@@ -220,15 +220,15 @@ get_metrics_as_dicts_with_nonexistent_key(Config) ->
     Result = get_metrics_as_dicts_with_keys([<<"not_existing">>], Config),
     ParsedResult = get_ok_value([data, metric, getMetricsAsDicts], Result),
     Map = dict_objects_to_map(ParsedResult),
-    SentName = [<<"global">>, <<"data">>, <<"xmpp">>, <<"received">>, <<"encrypted_size">>],
+    SentName = [<<"global">>, <<"data">>, <<"xmpp">>, <<"received">>, <<"xml_stanza_size">>],
     [] = maps:get(SentName, Map).
 
 get_metrics_as_dicts_empty_args(Config) ->
     %% Empty name
     Result = get_metrics_as_dicts([], [<<"median">>], Config),
     ParsedResult = get_ok_value([data, metric, getMetricsAsDicts], Result),
     Map = dict_objects_to_map(ParsedResult),
-    SentName = [<<"global">>, <<"data">>, <<"xmpp">>, <<"received">>, <<"encrypted_size">>],
+    SentName = [<<"global">>, <<"data">>, <<"xmpp">>, <<"received">>, <<"xml_stanza_size">>],
     [#{<<"key">> := <<"median">>, <<"value">> := Median}] = maps:get(SentName, Map),
     ?assert(is_integer(Median)),
     %% Empty keys
@@ -345,7 +345,7 @@ check_node_result_is_valid(ResList, MetricsAreGlobal) ->
         maps:get([<<"global">>,<<"uniqueSessionCount">>], Map),
     ?assert(is_integer(V)),
     HistObjects = maps:get([<<"global">>, <<"data">>, <<"xmpp">>,
-                            <<"sent">>, <<"compressed_size">>], Map),
+                            <<"sent">>, <<"xml_stanza_size">>], Map),
     check_histogram(kv_objects_to_map(HistObjects)).
 
 check_histogram(Map) ->

big_tests/tests/mim_c2s_SUITE.erl

@@ -64,7 +64,14 @@ end_per_testcase(Name, Config) ->
 %% tests
 %%--------------------------------------------------------------------
 two_users_can_log_and_chat(Config) ->
-    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
+    MongooseMetrics = [{[global, data, xmpp, received, xml_stanza_size], changed},
+                       {[global, data, xmpp, sent, xml_stanza_size], changed},
+                       {[global, data, xmpp, received, c2s, tcp, raw], changed},
+                       {[global, data, xmpp, sent, c2s, tcp, raw], changed},
+                       {[global, data, xmpp, received, c2s, tls, raw], 0},
+                       {[global, data, xmpp, sent, c2s, tls, raw], 0}],
+    escalus:fresh_story([{mongoose_metrics, MongooseMetrics} | Config],
+                        [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
         escalus_client:send(Alice, escalus_stanza:chat_to(Bob, <<"Hi!">>)),
         escalus:assert(is_chat_message, [<<"Hi!">>], escalus_client:wait_for_stanza(Bob)),
         escalus_client:send(Bob, escalus_stanza:chat_to(Alice, <<"Hi!">>)),

big_tests/tests/vcard_simple_SUITE.erl

@@ -456,7 +456,7 @@ configure_mod_vcard(Config) ->
 ldap_opts() ->
     LDAPOpts = #{filter => <<"(objectClass=inetOrgPerson)">>,
                  base => <<"ou=Users,dc=esl,dc=com">>,
-                 search_fields => [{"Full Name", "cn"}, {"User", "uid"}],
+                 search_fields => [{<<"Full Name">>, <<"cn">>}, {<<"User">>, <<"uid">>}],
                  vcard_map => [{"FN", "%s", ["cn"]}]},
     LDAPOptsWithDefaults = config_parser_helper:config([modules, mod_vcard, ldap], LDAPOpts),
     config_parser_helper:mod_config(mod_vcard, #{backend => ldap, ldap => LDAPOptsWithDefaults}).

big_tests/tests/websockets_SUITE.erl

@@ -27,7 +27,8 @@
 -define(REGISTRATION_TIMEOUT, 2).  %% seconds
 
 all() ->
-    [{group, ws_chat},
+    [metrics_test,
+     {group, ws_chat},
      {group, wss_chat}].
 
 groups() ->
@@ -47,23 +48,24 @@ suite() ->
 %%--------------------------------------------------------------------
 
 init_per_suite(Config) ->
-    escalus:init_per_suite(Config).
+    Config1 = escalus:init_per_suite(Config),
+    escalus:create_users(Config1, escalus:get_users([alice, geralt, geralt_s, carol])).
 
 end_per_suite(Config) ->
-    escalus:end_per_suite(Config).
+    Config1 = escalus:delete_users(Config, escalus:get_users([alice, geralt, geralt_s, carol])),
+    escalus:end_per_suite(Config1).
 
 init_per_group(GroupName, Config) ->
-    Config1 = escalus:create_users(Config, escalus:get_users([alice, geralt, geralt_s, carol])),
     case GroupName of
         wss_chat ->
-            [{user, geralt_s} | Config1];
+            [{user, geralt_s} | Config];
         _ ->
-            [{user, geralt} | Config1]
+            [{user, geralt} | Config]
     end.
 
 
-end_per_group(_GroupName, Config) ->
-    escalus:delete_users(Config, escalus:get_users([alice, geralt, geralt_s, carol])).
+end_per_group(_GroupName, _Config) ->
+    ok.
 
 init_per_testcase(CaseName, Config) ->
     escalus:init_per_testcase(CaseName, Config).
@@ -75,6 +77,25 @@ end_per_testcase(CaseName, Config) ->
 %% Message tests
 %%--------------------------------------------------------------------
 
+metrics_test(Config) ->
+    MongooseMetrics = [{[global, data, xmpp, received, xml_stanza_size], changed},
+                       {[global, data, xmpp, sent, xml_stanza_size], changed},
+                       {[global, data, xmpp, received, c2s, websocket, raw], changed},
+                       {[global, data, xmpp, sent, c2s, websocket, raw], changed},
+                       {[global, data, xmpp, received, c2s, tcp, raw], 0}, 
+                       {[global, data, xmpp, sent, c2s, tcp, raw], 0}],
+    escalus:story([{mongoose_metrics, MongooseMetrics} | Config],
+                  [{geralt, 1}, {geralt_s, 1}],
+                  fun(Geralt, GeraltS) ->
+
+        escalus_client:send(GeraltS, escalus_stanza:chat_to(Geralt, <<"Hi!">>)),
+        escalus:assert(is_chat_message, [<<"Hi!">>], escalus_client:wait_for_stanza(Geralt)),
+
+        escalus_client:send(Geralt, escalus_stanza:chat_to(GeraltS, <<"Hello!">>)),
+        escalus:assert(is_chat_message, [<<"Hello!">>], escalus_client:wait_for_stanza(GeraltS))
+
+        end).
+
 chat_msg(Config) ->
     escalus:story(Config, [{alice, 1}, {?config(user, Config), 1}, {carol, 1}],
                   fun(Alice, Geralt, Carol) ->

doc/operation-and-maintenance/MongooseIM-metrics.md

@@ -163,10 +163,6 @@ Metrics specific to an extension, e.g. Message Archive Management, are described
 | ----------- | ---- | ----------- |
 | `[global, data, xmpp, received, xml_stanza_size]` | histogram | A size (in bytes) of a received stanza after decompression and decryption. |
 | `[global, data, xmpp, sent, xml_stanza_size]` | histogram | A size (in bytes) of a stanza sent to a client socket. |
-| `[global, data, xmpp, received, compressed_size]` | histogram | A size (in bytes) of a received stanza before decompression. |
-| `[global, data, xmpp, sent, compressed_size]` | histogram | A size (in bytes) of a stanza after compression. |
-| `[global, data, xmpp, received, encrypted_size]` | histogram | A size (in bytes) of a received stanza before decryption. |
-| `[global, data, xmpp, sent, encrypted_size]` | histogram | A size (in bytes) of a stanza after encryption. |
 | `[global, data, dist]` | proplist | Network stats for an Erlang distributed communication. A proplist with values: `recv_oct`, `recv_cnt`, `recv_max`, `send_oct`, `send_max`, `send_cnt`, `send_pend`, `connections` |
 | `[global, data, rdbms, PoolName]` | proplist | For every RDBMS pool defined, an instance of this metric is available. It is a proplist with values `workers`, `recv_oct`, `recv_cnt`, `recv_max`, `send_oct`, `send_max`, `send_cnt`, `send_pend`. |
 

include/mongoose_ns.hrl

@@ -85,8 +85,6 @@
 -define(NS_FEATURE_COMPRESS,    <<"http://jabber.org/features/compress">>).
 -define(NS_FEATURE_MSGOFFLINE,  <<"msgoffline">>).
 
--define(NS_COMPRESS,            <<"http://jabber.org/protocol/compress">>).
-
 -define(NS_CAPS,                <<"http://jabber.org/protocol/caps">>).
 -define(NS_SHIM,                <<"http://jabber.org/protocol/shim">>).
 -define(NS_ADDRESS,             <<"http://jabber.org/protocol/address">>).

src/c2s/mongoose_c2s.erl

@@ -685,7 +685,7 @@ handle_stanza_from_client(#c2s_data{host_type = HostType}, HookParams, Acc, <<"m
     Acc1 = mongoose_c2s_hooks:user_send_message(HostType, Acc, HookParams),
     Acc2 = maybe_route(Acc1),
     TS1 = erlang:system_time(microsecond),
-    mongoose_metrics:update(HostType, [data, xmpp, sent, message, processing_time], (TS1 - TS0)),
+    mongoose_metrics:update(HostType, [data, xmpp, message, processing_time], (TS1 - TS0)),
     Acc2;
 handle_stanza_from_client(#c2s_data{host_type = HostType}, HookParams, Acc, <<"iq">>) ->
     Acc1 = mongoose_c2s_hooks:user_send_iq(HostType, Acc, HookParams),
@@ -937,8 +937,13 @@ do_send_element(StateData = #c2s_data{host_type = HostType}, #xmlel{} = El, Acc)
     mongoose_hooks:xmpp_send_element(HostType, Acc1, El).
 
 -spec send_xml(data(), exml_stream:element() | [exml_stream:element()]) -> maybe_ok().
-send_xml(#c2s_data{socket = Socket}, Xml) ->
-    mongoose_c2s_socket:send_xml(Socket, Xml).
+send_xml(Data, XmlElement) when is_tuple(XmlElement) ->
+    send_xml(Data, [XmlElement]);
+send_xml(#c2s_data{socket = Socket}, XmlElements) when is_list(XmlElements) ->
+    [mongoose_metrics:update(global, [data, xmpp, sent, xml_stanza_size], exml:xml_size(El))
+      || El <- XmlElements],
+    mongoose_c2s_socket:send_xml(Socket, XmlElements).
+
 
 state_timeout(#{c2s_state_timeout := Timeout}) ->
     {state_timeout, Timeout, state_timeout_termination}.

src/c2s/mongoose_c2s_ranch.erl

@@ -74,17 +74,19 @@ tcp_to_tls(just_tls, TcpSocket, TlsConfig) ->
 -spec socket_handle_data(state(), {tcp | ssl, term(), iodata()}) ->
   iodata() | {raw, [exml:element()]} | {error, term()}.
 socket_handle_data(#state{transport = fast_tls, socket = TlsSocket}, {tcp, _, Data}) ->
-    mongoose_metrics:update(global, [data, xmpp, received, encrypted_size], iolist_size(Data)),
     case fast_tls:recv_data(TlsSocket, Data) of
         {ok, DecryptedData} ->
+            DataSize = byte_size(DecryptedData),
+            mongoose_metrics:update(global, [data, xmpp, received, c2s, tls, raw], DataSize),
             DecryptedData;
         {error, Reason} ->
             {error, Reason}
     end;
 socket_handle_data(#state{transport = just_tls}, {ssl, _, Data}) ->
-    mongoose_metrics:update(global, [data, xmpp, received, encrypted_size], iolist_size(Data)),
+    mongoose_metrics:update(global, [data, xmpp, received, c2s, tls, raw], byte_size(Data)),
     Data;
 socket_handle_data(#state{transport = ranch_tcp, socket = Socket}, {tcp, Socket, Data}) ->
+    mongoose_metrics:update(global, [data, xmpp, received, c2s, tcp, raw], byte_size(Data)),
     Data.
 
 -spec socket_activate(state()) -> ok.
@@ -109,18 +111,20 @@ socket_send_xml(#state{transport = Transport, socket = Socket}, XML) ->
     Text = exml:to_iolist(XML),
     case send(Transport, Socket, Text) of
         ok ->
-            mongoose_metrics:update(global, [data, xmpp, sent, xml_stanza_size], iolist_size(Text)),
             ok;
         Error ->
             Error
         end.
 
 -spec send(transport(), ranch_transport:socket(), iodata()) -> ok | {error, term()}.
 send(fast_tls, Socket, Data) ->
+    mongoose_metrics:update(global, [data, xmpp, sent, c2s, tls, raw], iolist_size(Data)),
     fast_tls:send(Socket, Data);
 send(just_tls, Socket, Data) ->
+    mongoose_metrics:update(global, [data, xmpp, sent, c2s, tls, raw], iolist_size(Data)),
     just_tls:send(Socket, Data);
 send(ranch_tcp, Socket, Data) ->
+    mongoose_metrics:update(global, [data, xmpp, sent, c2s, tcp, raw], iolist_size(Data)),
     ranch_tcp:send(Socket, Data).
 
 -spec get_peer_certificate(state(), mongoose_listener:options()) ->

src/c2s/mongoose_c2s_socket.erl

@@ -29,11 +29,11 @@
 -callback socket_close(state()) -> ok.
 -callback socket_send_xml(state(), iodata() | exml_stream:element() | [exml_stream:element()]) ->
     ok | {error, term()}.
--callback get_peer_certificate(mongoose_c2s_socket:state(), mongoose_c2s:listener_opts()) -> peercert_return().
--callback has_peer_cert(mongoose_c2s_socket:state(), mongoose_c2s:listener_opts()) -> boolean().
--callback is_channel_binding_supported(mongoose_c2s_socket:state()) -> boolean().
--callback get_tls_last_message(mongoose_c2s_socket:state()) -> {ok, binary()} | {error, term()}.
--callback is_ssl(mongoose_c2s_socket:state()) -> boolean().
+-callback get_peer_certificate(state(), mongoose_c2s:listener_opts()) -> peercert_return().
+-callback has_peer_cert(state(), mongoose_c2s:listener_opts()) -> boolean().
+-callback is_channel_binding_supported(state()) -> boolean().
+-callback get_tls_last_message(state()) -> {ok, binary()} | {error, term()}.
+-callback is_ssl(state()) -> boolean().
 
 -record(c2s_socket, {module :: module(),
                      state :: state()}).

src/ejabberd_receiver.erl

@@ -201,8 +201,6 @@ handle_info({Tag, _TCPSocket, Data},
   when (Tag == tcp) or (Tag == ssl) ->
     case SockMod of
         mongoose_tls ->
-            mongoose_metrics:update(global,
-                            [data, xmpp, received, encrypted_size], size(Data)),
             case mongoose_tls:recv_data(Socket, Data) of
                 {ok, TLSData} ->
                     NewState = process_data(TLSData, State),

src/metrics/mongoose_metrics.erl

@@ -457,6 +457,8 @@ do_create_metric(PrefixedMetric, ExometerType, ExometerOpts) ->
 create_data_metrics() ->
     lists:foreach(fun(Metric) -> ensure_metric(global, Metric, histogram) end,
         ?GLOBAL_HISTOGRAMS),
+    lists:foreach(fun(Metric) -> ensure_metric(global, Metric, spiral) end,
+        ?GLOBAL_SPIRALS),
     lists:foreach(fun({Metric, Spec}) -> ensure_metric(global, Metric, Spec) end,
         ?DATA_FUN_METRICS).
 

src/metrics/mongoose_metrics_definitions.hrl

@@ -34,6 +34,18 @@
     modPrivacyStanzaAll
 ]).
 
+-define(GLOBAL_SPIRALS, [
+    %% TBD report raw data metrics for s2s and components conection
+    [data, xmpp, received, c2s, tcp, raw],
+    [data, xmpp, received, c2s, tls, raw],
+    [data, xmpp, received, c2s, bosh, raw],
+    [data, xmpp, received, c2s, websocket, raw],
+    [data, xmpp, sent, c2s, tcp, raw],
+    [data, xmpp, sent, c2s, tls, raw],
+    [data, xmpp, sent, c2s, bosh, raw],
+    [data, xmpp, sent, c2s, websocket, raw]
+]).
+
 -define(TOTAL_COUNTERS, [
     sessionCount
 ]).
@@ -66,13 +78,9 @@
                    {[erlang, memory], [function, erlang, memory, ['$dp'], value],
                     [total, processes_used, atom_used, binary, ets, system]}]).
 
--define(GLOBAL_HISTOGRAMS, [[data, xmpp, received, encrypted_size],
-                            [data, xmpp, received, compressed_size],
-                            [data, xmpp, received, xml_stanza_size],
-                            [data, xmpp, sent, encrypted_size],
-                            [data, xmpp, sent, compressed_size],
+-define(GLOBAL_HISTOGRAMS, [[data, xmpp, received, xml_stanza_size],
                             [data, xmpp, sent, xml_stanza_size],
-                            [data, xmpp, sent, message, processing_time]
+                            [data, xmpp, message, processing_time]
                            ]).
 
 -define(DATA_FUN_METRICS,

src/mod_bosh.erl

@@ -178,6 +178,9 @@ info(forward_body, Req, S) ->
     forward_body(Req1, BodyElem, S#rstate{req_sid = Sid});
 info({bosh_reply, El}, Req, S) ->
     BEl = exml:to_binary(El),
+    %% for BOSH 'data.xmpp.sent.raw' metric includes 'body' wrapping elements
+    %% and resending attempts
+    mongoose_metrics:update(global, [data, xmpp, sent, c2s, bosh, raw], byte_size(BEl)),
     ?LOG_DEBUG(#{what => bosh_send, req_sid => S#rstate.req_sid, reply_body => BEl,
                  sid => exml_query:attr(El, <<"sid">>, <<"missing">>)}),
     Headers = bosh_reply_headers(),
@@ -301,6 +304,8 @@ forward_body(Req, #xmlel{} = Body, S) ->
 
 -spec handle_request(pid(), event_type(), exml:element()) -> ok.
 handle_request(Socket, EventType, Body) ->
+    %% for BOSH 'data.xmpp.received.raw' metric includes 'body' wrapping elements
+    mongoose_metrics:update(global, [data, xmpp, received, c2s, bosh, raw], exml:xml_size(Body)),
     mod_bosh_socket:handle_request(Socket, {EventType, self(), Body}).