From 1289b1f08ca950a33b8a9f64a80420acef88740b Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Wed, 15 Feb 2023 15:23:00 +0100 Subject: [PATCH 01/12] removing data.xmpp.*.compressed_size metrics --- big_tests/tests/graphql_metric_SUITE.erl | 6 +++--- doc/operation-and-maintenance/MongooseIM-metrics.md | 2 -- include/mongoose_ns.hrl | 2 -- src/metrics/mongoose_metrics_definitions.hrl | 2 -- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/big_tests/tests/graphql_metric_SUITE.erl b/big_tests/tests/graphql_metric_SUITE.erl index 02de737057..3326dfaac7 100644 --- a/big_tests/tests/graphql_metric_SUITE.erl +++ b/big_tests/tests/graphql_metric_SUITE.erl @@ -220,7 +220,7 @@ get_metrics_as_dicts_with_nonexistent_key(Config) -> Result = get_metrics_as_dicts_with_keys([<<"not_existing">>], Config), ParsedResult = get_ok_value([data, metric, getMetricsAsDicts], Result), Map = dict_objects_to_map(ParsedResult), - SentName = [<<"global">>, <<"data">>, <<"xmpp">>, <<"received">>, <<"encrypted_size">>], + SentName = [<<"global">>, <<"data">>, <<"xmpp">>, <<"received">>, <<"xml_stanza_size">>], [] = maps:get(SentName, Map). get_metrics_as_dicts_empty_args(Config) -> @@ -228,7 +228,7 @@ get_metrics_as_dicts_empty_args(Config) -> Result = get_metrics_as_dicts([], [<<"median">>], Config), ParsedResult = get_ok_value([data, metric, getMetricsAsDicts], Result), Map = dict_objects_to_map(ParsedResult), - SentName = [<<"global">>, <<"data">>, <<"xmpp">>, <<"received">>, <<"encrypted_size">>], + SentName = [<<"global">>, <<"data">>, <<"xmpp">>, <<"received">>, <<"xml_stanza_size">>], [#{<<"key">> := <<"median">>, <<"value">> := Median}] = maps:get(SentName, Map), ?assert(is_integer(Median)), %% Empty keys @@ -345,7 +345,7 @@ check_node_result_is_valid(ResList, MetricsAreGlobal) -> maps:get([<<"global">>,<<"uniqueSessionCount">>], Map), ?assert(is_integer(V)), HistObjects = maps:get([<<"global">>, <<"data">>, <<"xmpp">>, - <<"sent">>, <<"compressed_size">>], Map), + <<"sent">>, <<"xml_stanza_size">>], Map), check_histogram(kv_objects_to_map(HistObjects)). check_histogram(Map) -> diff --git a/doc/operation-and-maintenance/MongooseIM-metrics.md b/doc/operation-and-maintenance/MongooseIM-metrics.md index 1b8f9a214d..a5f8182c4f 100644 --- a/doc/operation-and-maintenance/MongooseIM-metrics.md +++ b/doc/operation-and-maintenance/MongooseIM-metrics.md @@ -163,8 +163,6 @@ Metrics specific to an extension, e.g. Message Archive Management, are described | ----------- | ---- | ----------- | | `[global, data, xmpp, received, xml_stanza_size]` | histogram | A size (in bytes) of a received stanza after decompression and decryption. | | `[global, data, xmpp, sent, xml_stanza_size]` | histogram | A size (in bytes) of a stanza sent to a client socket. | -| `[global, data, xmpp, received, compressed_size]` | histogram | A size (in bytes) of a received stanza before decompression. | -| `[global, data, xmpp, sent, compressed_size]` | histogram | A size (in bytes) of a stanza after compression. | | `[global, data, xmpp, received, encrypted_size]` | histogram | A size (in bytes) of a received stanza before decryption. | | `[global, data, xmpp, sent, encrypted_size]` | histogram | A size (in bytes) of a stanza after encryption. | | `[global, data, dist]` | proplist | Network stats for an Erlang distributed communication. A proplist with values: `recv_oct`, `recv_cnt`, `recv_max`, `send_oct`, `send_max`, `send_cnt`, `send_pend`, `connections` | diff --git a/include/mongoose_ns.hrl b/include/mongoose_ns.hrl index 49e11a0eeb..4fe7f50baf 100644 --- a/include/mongoose_ns.hrl +++ b/include/mongoose_ns.hrl @@ -85,8 +85,6 @@ -define(NS_FEATURE_COMPRESS, <<"http://jabber.org/features/compress">>). -define(NS_FEATURE_MSGOFFLINE, <<"msgoffline">>). --define(NS_COMPRESS, <<"http://jabber.org/protocol/compress">>). - -define(NS_CAPS, <<"http://jabber.org/protocol/caps">>). -define(NS_SHIM, <<"http://jabber.org/protocol/shim">>). -define(NS_ADDRESS, <<"http://jabber.org/protocol/address">>). diff --git a/src/metrics/mongoose_metrics_definitions.hrl b/src/metrics/mongoose_metrics_definitions.hrl index 3ff9208895..9272fd003f 100644 --- a/src/metrics/mongoose_metrics_definitions.hrl +++ b/src/metrics/mongoose_metrics_definitions.hrl @@ -67,10 +67,8 @@ [total, processes_used, atom_used, binary, ets, system]}]). -define(GLOBAL_HISTOGRAMS, [[data, xmpp, received, encrypted_size], - [data, xmpp, received, compressed_size], [data, xmpp, received, xml_stanza_size], [data, xmpp, sent, encrypted_size], - [data, xmpp, sent, compressed_size], [data, xmpp, sent, xml_stanza_size], [data, xmpp, sent, message, processing_time] ]). From c82583f9c006b0a6e02ebd714c10251ca71c9ff1 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Wed, 15 Feb 2023 17:13:12 +0100 Subject: [PATCH 02/12] renaming global.data.xmpp.sent.message.processing_time metric into global.data.xmpp.message.processing_time --- src/c2s/mongoose_c2s.erl | 2 +- src/metrics/mongoose_metrics_definitions.hrl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/c2s/mongoose_c2s.erl b/src/c2s/mongoose_c2s.erl index 37adcd04e6..3da69607b7 100644 --- a/src/c2s/mongoose_c2s.erl +++ b/src/c2s/mongoose_c2s.erl @@ -685,7 +685,7 @@ handle_stanza_from_client(#c2s_data{host_type = HostType}, HookParams, Acc, <<"m Acc1 = mongoose_c2s_hooks:user_send_message(HostType, Acc, HookParams), Acc2 = maybe_route(Acc1), TS1 = erlang:system_time(microsecond), - mongoose_metrics:update(HostType, [data, xmpp, sent, message, processing_time], (TS1 - TS0)), + mongoose_metrics:update(HostType, [data, xmpp, message, processing_time], (TS1 - TS0)), Acc2; handle_stanza_from_client(#c2s_data{host_type = HostType}, HookParams, Acc, <<"iq">>) -> Acc1 = mongoose_c2s_hooks:user_send_iq(HostType, Acc, HookParams), diff --git a/src/metrics/mongoose_metrics_definitions.hrl b/src/metrics/mongoose_metrics_definitions.hrl index 9272fd003f..1744feff2b 100644 --- a/src/metrics/mongoose_metrics_definitions.hrl +++ b/src/metrics/mongoose_metrics_definitions.hrl @@ -70,7 +70,7 @@ [data, xmpp, received, xml_stanza_size], [data, xmpp, sent, encrypted_size], [data, xmpp, sent, xml_stanza_size], - [data, xmpp, sent, message, processing_time] + [data, xmpp, message, processing_time] ]). -define(DATA_FUN_METRICS, From c5f8d4595026b6055d3bddde8895c041ffcc31a1 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Sun, 19 Feb 2023 02:44:56 +0100 Subject: [PATCH 03/12] removing ejabberd_socket compatibility from mod_bosh_socket --- src/c2s/mongoose_c2s_socket.erl | 10 ++-- src/mod_bosh_socket.erl | 94 +++++---------------------------- 2 files changed, 19 insertions(+), 85 deletions(-) diff --git a/src/c2s/mongoose_c2s_socket.erl b/src/c2s/mongoose_c2s_socket.erl index f77b6bfce5..d9afdfce90 100644 --- a/src/c2s/mongoose_c2s_socket.erl +++ b/src/c2s/mongoose_c2s_socket.erl @@ -29,11 +29,11 @@ -callback socket_close(state()) -> ok. -callback socket_send_xml(state(), iodata() | exml_stream:element() | [exml_stream:element()]) -> ok | {error, term()}. --callback get_peer_certificate(mongoose_c2s_socket:state(), mongoose_c2s:listener_opts()) -> peercert_return(). --callback has_peer_cert(mongoose_c2s_socket:state(), mongoose_c2s:listener_opts()) -> boolean(). --callback is_channel_binding_supported(mongoose_c2s_socket:state()) -> boolean(). --callback get_tls_last_message(mongoose_c2s_socket:state()) -> {ok, binary()} | {error, term()}. --callback is_ssl(mongoose_c2s_socket:state()) -> boolean(). +-callback get_peer_certificate(state(), mongoose_c2s:listener_opts()) -> peercert_return(). +-callback has_peer_cert(state(), mongoose_c2s:listener_opts()) -> boolean(). +-callback is_channel_binding_supported(state()) -> boolean(). +-callback get_tls_last_message(state()) -> {ok, binary()} | {error, term()}. +-callback is_ssl(state()) -> boolean(). -record(c2s_socket, {module :: module(), state :: state()}). diff --git a/src/mod_bosh_socket.erl b/src/mod_bosh_socket.erl index 93e589a190..2ad6c6854f 100644 --- a/src/mod_bosh_socket.erl +++ b/src/mod_bosh_socket.erl @@ -1,7 +1,6 @@ -module(mod_bosh_socket). -behaviour(gen_fsm_compat). --behaviour(mongoose_transport). -behaviour(mongoose_c2s_socket). %% API @@ -19,16 +18,6 @@ set_client_acks/2, get_cached_responses/1]). -%% ejabberd_socket compatibility --export([starttls/2, starttls/3, - send/2, - send_xml/2, - change_shaper/2, - monitor/1, - get_sockmod/1, - close/1, - peername/1, - get_peer_certificate/1]). %% mongoose_c2s_socket callbacks -export([socket_new/2, @@ -1026,68 +1015,6 @@ is_stream_prefix(#xmlel{name = <<"stream:error">>}) -> true; is_stream_prefix(#xmlel{name = <<"stream:features">>}) -> true; is_stream_prefix(_) -> false. -%%-------------------------------------------------------------------- -%% ejabberd_socket compatibility -%%-------------------------------------------------------------------- - -%% @doc Should be negotiated on HTTP level. --spec starttls(mod_bosh:socket(), _) -> no_return(). -starttls(SocketData, TLSOpts) -> - starttls(SocketData, TLSOpts, <<>>). - - --spec starttls(mod_bosh:socket(), _, _) -> no_return(). -starttls(_SocketData, _TLSOpts, _Data) -> - throw({error, negotiate_tls_on_http_level}). - - --spec send_xml(mod_bosh:socket(), mongoose_transport:send_xml_input()) -> ok. -send_xml(Socket, {xmlstreamelement, XML}) -> - send(Socket, XML); -send_xml(Socket, #xmlstreamstart{} = XML) -> - send(Socket, XML); -send_xml(Socket, #xmlstreamend{} = XML) -> - send(Socket, XML). - - --spec send(mod_bosh:socket(), _) -> 'ok'. -send(#bosh_socket{pid = Pid}, Data) -> - Pid ! {send, Data}, - ok. - --spec change_shaper(mod_bosh:socket(), shaper:shaper()) -> mod_bosh:socket(). -change_shaper(SocketData, _Shaper) -> - %% TODO: we ignore shapers for now - SocketData. - - --spec monitor(mod_bosh:socket()) -> reference(). -monitor(#bosh_socket{pid = Pid}) -> - erlang:monitor(process, Pid). - - --spec get_sockmod(mod_bosh:socket()) -> module(). -get_sockmod(_SocketData) -> - ?MODULE. - - --spec close(mod_bosh:socket()) -> 'close'. -close(#bosh_socket{pid = Pid}) -> - Pid ! close. - --spec peername(mod_bosh:socket()) -> mongoose_transport:peername_return(). -peername(#bosh_socket{peer = Peer}) -> - {ok, Peer}. - -get_peer_certificate(S, _) -> - get_peer_certificate(S). - --spec get_peer_certificate(mod_bosh:socket()) -> mongoose_transport:peercert_return(). -get_peer_certificate(#bosh_socket{peercert = undefined}) -> - no_peer_cert; -get_peer_certificate(#bosh_socket{peercert = PeerCert}) -> - Decoded = public_key:pkix_decode_cert(PeerCert, plain), - {ok, Decoded}. %%-------------------------------------------------------------------- %% Helpers @@ -1125,9 +1052,8 @@ socket_new(Socket, _LOpts) -> Socket. -spec socket_peername(mod_bosh:socket()) -> {inet:ip_address(), inet:port_number()}. -socket_peername(Socket) -> - {ok, Peername} = peername(Socket), - Peername. +socket_peername(#bosh_socket{peer = Peer}) -> + Peer. -spec tcp_to_tls(mod_bosh:socket(), mongoose_listener:options()) -> {ok, mod_bosh:socket()} | {error, term()}. @@ -1154,13 +1080,21 @@ socket_send_xml(#bosh_socket{pid = Pid}, XML) -> ok. -spec socket_close(mod_bosh:socket()) -> ok. -socket_close(Socket) -> - close(Socket), +socket_close(#bosh_socket{pid = Pid}) -> + Pid ! close, ok. +-spec get_peer_certificate(mod_bosh:socket(), mongoose_listener:options()) -> + mongoose_transport:peercert_return(). +get_peer_certificate(#bosh_socket{peercert = undefined}, _) -> + no_peer_cert; +get_peer_certificate(#bosh_socket{peercert = PeerCert}, _) -> + Decoded = public_key:pkix_decode_cert(PeerCert, plain), + {ok, Decoded}. + -spec has_peer_cert(mod_bosh:socket(), mongoose_listener:options()) -> boolean(). -has_peer_cert(Socket, _LOpts) -> - get_peer_certificate(Socket) /= no_peer_cert. +has_peer_cert(Socket, LOpts) -> + get_peer_certificate(Socket, LOpts) /= no_peer_cert. -spec is_channel_binding_supported(mod_bosh:socket()) -> boolean(). is_channel_binding_supported(_Socket) -> From c8fc048d4a50024880df2951122ffbb7d095da30 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Sun, 19 Feb 2023 03:49:02 +0100 Subject: [PATCH 04/12] moving 'data.xmpp.sent.xml_stanza_size' metric calculation to mongoose_c2s --- src/c2s/mongoose_c2s.erl | 9 +++++++-- src/c2s/mongoose_c2s_ranch.erl | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/c2s/mongoose_c2s.erl b/src/c2s/mongoose_c2s.erl index 3da69607b7..a39828942a 100644 --- a/src/c2s/mongoose_c2s.erl +++ b/src/c2s/mongoose_c2s.erl @@ -937,8 +937,13 @@ do_send_element(StateData = #c2s_data{host_type = HostType}, #xmlel{} = El, Acc) mongoose_hooks:xmpp_send_element(HostType, Acc1, El). -spec send_xml(data(), exml_stream:element() | [exml_stream:element()]) -> maybe_ok(). -send_xml(#c2s_data{socket = Socket}, Xml) -> - mongoose_c2s_socket:send_xml(Socket, Xml). +send_xml(Data, XmlElement) when is_tuple(XmlElement) -> + send_xml(Data, [XmlElement]); +send_xml(#c2s_data{socket = Socket}, XmlElements) when is_list(XmlElements) -> + [mongoose_metrics:update(global, [data, xmpp, sent, xml_stanza_size], exml:xml_size(El)) + || El <- XmlElements], + mongoose_c2s_socket:send_xml(Socket, XmlElements). + state_timeout(#{c2s_state_timeout := Timeout}) -> {state_timeout, Timeout, state_timeout_termination}. diff --git a/src/c2s/mongoose_c2s_ranch.erl b/src/c2s/mongoose_c2s_ranch.erl index 52fb23b38c..007b5b3967 100644 --- a/src/c2s/mongoose_c2s_ranch.erl +++ b/src/c2s/mongoose_c2s_ranch.erl @@ -109,7 +109,6 @@ socket_send_xml(#state{transport = Transport, socket = Socket}, XML) -> Text = exml:to_iolist(XML), case send(Transport, Socket, Text) of ok -> - mongoose_metrics:update(global, [data, xmpp, sent, xml_stanza_size], iolist_size(Text)), ok; Error -> Error From 2dce49c2d37c24181fc44c5d801c462d3be37d64 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Sun, 19 Feb 2023 12:27:38 +0100 Subject: [PATCH 05/12] removing 'global.data.xmpp.*.encrypted_size' metrics --- doc/operation-and-maintenance/MongooseIM-metrics.md | 2 -- src/c2s/mongoose_c2s_ranch.erl | 2 -- src/ejabberd_receiver.erl | 2 -- src/metrics/mongoose_metrics_definitions.hrl | 4 +--- 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/doc/operation-and-maintenance/MongooseIM-metrics.md b/doc/operation-and-maintenance/MongooseIM-metrics.md index a5f8182c4f..45889a4fd3 100644 --- a/doc/operation-and-maintenance/MongooseIM-metrics.md +++ b/doc/operation-and-maintenance/MongooseIM-metrics.md @@ -163,8 +163,6 @@ Metrics specific to an extension, e.g. Message Archive Management, are described | ----------- | ---- | ----------- | | `[global, data, xmpp, received, xml_stanza_size]` | histogram | A size (in bytes) of a received stanza after decompression and decryption. | | `[global, data, xmpp, sent, xml_stanza_size]` | histogram | A size (in bytes) of a stanza sent to a client socket. | -| `[global, data, xmpp, received, encrypted_size]` | histogram | A size (in bytes) of a received stanza before decryption. | -| `[global, data, xmpp, sent, encrypted_size]` | histogram | A size (in bytes) of a stanza after encryption. | | `[global, data, dist]` | proplist | Network stats for an Erlang distributed communication. A proplist with values: `recv_oct`, `recv_cnt`, `recv_max`, `send_oct`, `send_max`, `send_cnt`, `send_pend`, `connections` | | `[global, data, rdbms, PoolName]` | proplist | For every RDBMS pool defined, an instance of this metric is available. It is a proplist with values `workers`, `recv_oct`, `recv_cnt`, `recv_max`, `send_oct`, `send_max`, `send_cnt`, `send_pend`. | diff --git a/src/c2s/mongoose_c2s_ranch.erl b/src/c2s/mongoose_c2s_ranch.erl index 007b5b3967..e4d6a16918 100644 --- a/src/c2s/mongoose_c2s_ranch.erl +++ b/src/c2s/mongoose_c2s_ranch.erl @@ -74,7 +74,6 @@ tcp_to_tls(just_tls, TcpSocket, TlsConfig) -> -spec socket_handle_data(state(), {tcp | ssl, term(), iodata()}) -> iodata() | {raw, [exml:element()]} | {error, term()}. socket_handle_data(#state{transport = fast_tls, socket = TlsSocket}, {tcp, _, Data}) -> - mongoose_metrics:update(global, [data, xmpp, received, encrypted_size], iolist_size(Data)), case fast_tls:recv_data(TlsSocket, Data) of {ok, DecryptedData} -> DecryptedData; @@ -82,7 +81,6 @@ socket_handle_data(#state{transport = fast_tls, socket = TlsSocket}, {tcp, _, Da {error, Reason} end; socket_handle_data(#state{transport = just_tls}, {ssl, _, Data}) -> - mongoose_metrics:update(global, [data, xmpp, received, encrypted_size], iolist_size(Data)), Data; socket_handle_data(#state{transport = ranch_tcp, socket = Socket}, {tcp, Socket, Data}) -> Data. diff --git a/src/ejabberd_receiver.erl b/src/ejabberd_receiver.erl index d44f128106..5ab9e75690 100644 --- a/src/ejabberd_receiver.erl +++ b/src/ejabberd_receiver.erl @@ -201,8 +201,6 @@ handle_info({Tag, _TCPSocket, Data}, when (Tag == tcp) or (Tag == ssl) -> case SockMod of mongoose_tls -> - mongoose_metrics:update(global, - [data, xmpp, received, encrypted_size], size(Data)), case mongoose_tls:recv_data(Socket, Data) of {ok, TLSData} -> NewState = process_data(TLSData, State), diff --git a/src/metrics/mongoose_metrics_definitions.hrl b/src/metrics/mongoose_metrics_definitions.hrl index 1744feff2b..c1dcd36d39 100644 --- a/src/metrics/mongoose_metrics_definitions.hrl +++ b/src/metrics/mongoose_metrics_definitions.hrl @@ -66,9 +66,7 @@ {[erlang, memory], [function, erlang, memory, ['$dp'], value], [total, processes_used, atom_used, binary, ets, system]}]). --define(GLOBAL_HISTOGRAMS, [[data, xmpp, received, encrypted_size], - [data, xmpp, received, xml_stanza_size], - [data, xmpp, sent, encrypted_size], +-define(GLOBAL_HISTOGRAMS, [[data, xmpp, received, xml_stanza_size], [data, xmpp, sent, xml_stanza_size], [data, xmpp, message, processing_time] ]). From 7b16adfd5cae0590db87ac03c7cb5f868ba99c54 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Sun, 19 Feb 2023 14:43:57 +0100 Subject: [PATCH 06/12] adding 'data.xmpp.*.raw' metrics --- src/c2s/mongoose_c2s_ranch.erl | 5 +++++ src/metrics/mongoose_metrics.erl | 2 ++ src/metrics/mongoose_metrics_definitions.hrl | 5 +++++ src/mod_bosh.erl | 5 +++++ src/mod_websockets.erl | 3 +++ 5 files changed, 20 insertions(+) diff --git a/src/c2s/mongoose_c2s_ranch.erl b/src/c2s/mongoose_c2s_ranch.erl index e4d6a16918..3c5e290b74 100644 --- a/src/c2s/mongoose_c2s_ranch.erl +++ b/src/c2s/mongoose_c2s_ranch.erl @@ -76,13 +76,17 @@ tcp_to_tls(just_tls, TcpSocket, TlsConfig) -> socket_handle_data(#state{transport = fast_tls, socket = TlsSocket}, {tcp, _, Data}) -> case fast_tls:recv_data(TlsSocket, Data) of {ok, DecryptedData} -> + DataSize = byte_size(DecryptedData), + mongoose_metrics:update(global, [data, xmpp, received, raw], DataSize), DecryptedData; {error, Reason} -> {error, Reason} end; socket_handle_data(#state{transport = just_tls}, {ssl, _, Data}) -> + mongoose_metrics:update(global, [data, xmpp, received, raw], byte_size(Data)), Data; socket_handle_data(#state{transport = ranch_tcp, socket = Socket}, {tcp, Socket, Data}) -> + mongoose_metrics:update(global, [data, xmpp, received, raw], byte_size(Data)), Data. -spec socket_activate(state()) -> ok. @@ -107,6 +111,7 @@ socket_send_xml(#state{transport = Transport, socket = Socket}, XML) -> Text = exml:to_iolist(XML), case send(Transport, Socket, Text) of ok -> + mongoose_metrics:update(global, [data, xmpp, sent, raw], iolist_size(Text)), ok; Error -> Error diff --git a/src/metrics/mongoose_metrics.erl b/src/metrics/mongoose_metrics.erl index 1b032e7650..e777889720 100644 --- a/src/metrics/mongoose_metrics.erl +++ b/src/metrics/mongoose_metrics.erl @@ -457,6 +457,8 @@ do_create_metric(PrefixedMetric, ExometerType, ExometerOpts) -> create_data_metrics() -> lists:foreach(fun(Metric) -> ensure_metric(global, Metric, histogram) end, ?GLOBAL_HISTOGRAMS), + lists:foreach(fun(Metric) -> ensure_metric(global, Metric, spiral) end, + ?GLOBAL_SPIRALS), lists:foreach(fun({Metric, Spec}) -> ensure_metric(global, Metric, Spec) end, ?DATA_FUN_METRICS). diff --git a/src/metrics/mongoose_metrics_definitions.hrl b/src/metrics/mongoose_metrics_definitions.hrl index c1dcd36d39..769a3ebf0d 100644 --- a/src/metrics/mongoose_metrics_definitions.hrl +++ b/src/metrics/mongoose_metrics_definitions.hrl @@ -34,6 +34,11 @@ modPrivacyStanzaAll ]). +-define(GLOBAL_SPIRALS, [ + [data, xmpp, received, raw], + [data, xmpp, sent, raw] +]). + -define(TOTAL_COUNTERS, [ sessionCount ]). diff --git a/src/mod_bosh.erl b/src/mod_bosh.erl index 32a1a8f887..f4ab448de9 100644 --- a/src/mod_bosh.erl +++ b/src/mod_bosh.erl @@ -178,6 +178,9 @@ info(forward_body, Req, S) -> forward_body(Req1, BodyElem, S#rstate{req_sid = Sid}); info({bosh_reply, El}, Req, S) -> BEl = exml:to_binary(El), + %% for BOSH 'data.xmpp.sent.raw' metric includes 'body' wrapping elements + %% and resending attempts + mongoose_metrics:update(global, [data, xmpp, sent, raw], byte_size(BEl)), ?LOG_DEBUG(#{what => bosh_send, req_sid => S#rstate.req_sid, reply_body => BEl, sid => exml_query:attr(El, <<"sid">>, <<"missing">>)}), Headers = bosh_reply_headers(), @@ -301,6 +304,8 @@ forward_body(Req, #xmlel{} = Body, S) -> -spec handle_request(pid(), event_type(), exml:element()) -> ok. handle_request(Socket, EventType, Body) -> + %% for BOSH 'data.xmpp.received.raw' metric includes 'body' wrapping elements + mongoose_metrics:update(global, [data, xmpp, received, raw], exml:xml_size(Body)), mod_bosh_socket:handle_request(Socket, {EventType, self(), Body}). diff --git a/src/mod_websockets.erl b/src/mod_websockets.erl index 63c6fc6cfc..07efb0eb6e 100644 --- a/src/mod_websockets.erl +++ b/src/mod_websockets.erl @@ -157,10 +157,12 @@ websocket_handle(Any, State) -> websocket_info({send, Text}, State) -> ?LOG_DEBUG(#{what => ws_send, text => <<"Sending text over WebSockets">>, msg => Text, peer => State#ws_state.peer}), + mongoose_metrics:update(global, [data, xmpp, sent, raw], iolist_size(Text)), {reply, {text, Text}, State}; websocket_info({send_xml, XML}, State) -> XML1 = process_server_stream_root(replace_stream_ns(XML, State), State), Text = exml:to_iolist(XML1), + mongoose_metrics:update(global, [data, xmpp, sent, raw], iolist_size(Text)), ?LOG_DEBUG(#{what => ws_send, text => <<"Sending xml over WebSockets">>, packet => Text, peer => State#ws_state.peer}), {reply, {text, Text}, State}; @@ -199,6 +201,7 @@ handle_text(Text, #ws_state{ parser = undefined } = State) -> handle_text(Text, #ws_state{parser = Parser} = State) -> case exml_stream:parse(Parser, Text) of {ok, NewParser, Elements} -> + mongoose_metrics:update(global, [data, xmpp, received, raw], byte_size(Text)), State1 = State#ws_state{ parser = NewParser }, case maybe_start_fsm(Elements, State1) of {ok, State2} -> From ada662560ad6b5d87dd0599dd8eec0f01fb04318 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Mon, 20 Feb 2023 05:27:35 +0100 Subject: [PATCH 07/12] reporting raw data metrics per transport type --- src/c2s/mongoose_c2s_ranch.erl | 10 ++++++---- src/metrics/mongoose_metrics_definitions.hrl | 11 +++++++++-- src/mod_bosh.erl | 4 ++-- src/mod_websockets.erl | 7 ++++--- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/c2s/mongoose_c2s_ranch.erl b/src/c2s/mongoose_c2s_ranch.erl index 3c5e290b74..0b26ddebab 100644 --- a/src/c2s/mongoose_c2s_ranch.erl +++ b/src/c2s/mongoose_c2s_ranch.erl @@ -77,16 +77,16 @@ socket_handle_data(#state{transport = fast_tls, socket = TlsSocket}, {tcp, _, Da case fast_tls:recv_data(TlsSocket, Data) of {ok, DecryptedData} -> DataSize = byte_size(DecryptedData), - mongoose_metrics:update(global, [data, xmpp, received, raw], DataSize), + mongoose_metrics:update(global, [data, xmpp, received, c2s, tls, raw], DataSize), DecryptedData; {error, Reason} -> {error, Reason} end; socket_handle_data(#state{transport = just_tls}, {ssl, _, Data}) -> - mongoose_metrics:update(global, [data, xmpp, received, raw], byte_size(Data)), + mongoose_metrics:update(global, [data, xmpp, received, c2s, tls, raw], byte_size(Data)), Data; socket_handle_data(#state{transport = ranch_tcp, socket = Socket}, {tcp, Socket, Data}) -> - mongoose_metrics:update(global, [data, xmpp, received, raw], byte_size(Data)), + mongoose_metrics:update(global, [data, xmpp, received, c2s, tcp, raw], byte_size(Data)), Data. -spec socket_activate(state()) -> ok. @@ -111,7 +111,6 @@ socket_send_xml(#state{transport = Transport, socket = Socket}, XML) -> Text = exml:to_iolist(XML), case send(Transport, Socket, Text) of ok -> - mongoose_metrics:update(global, [data, xmpp, sent, raw], iolist_size(Text)), ok; Error -> Error @@ -119,10 +118,13 @@ socket_send_xml(#state{transport = Transport, socket = Socket}, XML) -> -spec send(transport(), ranch_transport:socket(), iodata()) -> ok | {error, term()}. send(fast_tls, Socket, Data) -> + mongoose_metrics:update(global, [data, xmpp, sent, c2s, tls, raw], iolist_size(Data)), fast_tls:send(Socket, Data); send(just_tls, Socket, Data) -> + mongoose_metrics:update(global, [data, xmpp, sent, c2s, tls, raw], iolist_size(Data)), just_tls:send(Socket, Data); send(ranch_tcp, Socket, Data) -> + mongoose_metrics:update(global, [data, xmpp, sent, c2s, tcp, raw], iolist_size(Data)), ranch_tcp:send(Socket, Data). -spec get_peer_certificate(state(), mongoose_listener:options()) -> diff --git a/src/metrics/mongoose_metrics_definitions.hrl b/src/metrics/mongoose_metrics_definitions.hrl index 769a3ebf0d..65bae20709 100644 --- a/src/metrics/mongoose_metrics_definitions.hrl +++ b/src/metrics/mongoose_metrics_definitions.hrl @@ -35,8 +35,15 @@ ]). -define(GLOBAL_SPIRALS, [ - [data, xmpp, received, raw], - [data, xmpp, sent, raw] + %% TBD report raw data metrics for s2s and components conection + [data, xmpp, received, c2s, tcp, raw], + [data, xmpp, received, c2s, tls, raw], + [data, xmpp, received, c2s, bosh, raw], + [data, xmpp, received, c2s, websocket, raw], + [data, xmpp, sent, c2s, tcp, raw], + [data, xmpp, sent, c2s, tls, raw], + [data, xmpp, sent, c2s, bosh, raw], + [data, xmpp, sent, c2s, websocket, raw] ]). -define(TOTAL_COUNTERS, [ diff --git a/src/mod_bosh.erl b/src/mod_bosh.erl index f4ab448de9..8bfb7c073c 100644 --- a/src/mod_bosh.erl +++ b/src/mod_bosh.erl @@ -180,7 +180,7 @@ info({bosh_reply, El}, Req, S) -> BEl = exml:to_binary(El), %% for BOSH 'data.xmpp.sent.raw' metric includes 'body' wrapping elements %% and resending attempts - mongoose_metrics:update(global, [data, xmpp, sent, raw], byte_size(BEl)), + mongoose_metrics:update(global, [data, xmpp, sent, c2s, bosh, raw], byte_size(BEl)), ?LOG_DEBUG(#{what => bosh_send, req_sid => S#rstate.req_sid, reply_body => BEl, sid => exml_query:attr(El, <<"sid">>, <<"missing">>)}), Headers = bosh_reply_headers(), @@ -305,7 +305,7 @@ forward_body(Req, #xmlel{} = Body, S) -> -spec handle_request(pid(), event_type(), exml:element()) -> ok. handle_request(Socket, EventType, Body) -> %% for BOSH 'data.xmpp.received.raw' metric includes 'body' wrapping elements - mongoose_metrics:update(global, [data, xmpp, received, raw], exml:xml_size(Body)), + mongoose_metrics:update(global, [data, xmpp, received, c2s, bosh, raw], exml:xml_size(Body)), mod_bosh_socket:handle_request(Socket, {EventType, self(), Body}). diff --git a/src/mod_websockets.erl b/src/mod_websockets.erl index 07efb0eb6e..1f4f3597b5 100644 --- a/src/mod_websockets.erl +++ b/src/mod_websockets.erl @@ -34,6 +34,7 @@ get_tls_last_message/1, is_ssl/1]). +%% TBD: remove mongoose_transport compatibility %% ejabberd_socket compatibility -export([starttls/2, starttls/3, send/2, @@ -157,12 +158,12 @@ websocket_handle(Any, State) -> websocket_info({send, Text}, State) -> ?LOG_DEBUG(#{what => ws_send, text => <<"Sending text over WebSockets">>, msg => Text, peer => State#ws_state.peer}), - mongoose_metrics:update(global, [data, xmpp, sent, raw], iolist_size(Text)), + mongoose_metrics:update(global, [data, xmpp, sent, c2s, websocket, raw], iolist_size(Text)), {reply, {text, Text}, State}; websocket_info({send_xml, XML}, State) -> XML1 = process_server_stream_root(replace_stream_ns(XML, State), State), Text = exml:to_iolist(XML1), - mongoose_metrics:update(global, [data, xmpp, sent, raw], iolist_size(Text)), + mongoose_metrics:update(global, [data, xmpp, sent, c2s, websocket, raw], iolist_size(Text)), ?LOG_DEBUG(#{what => ws_send, text => <<"Sending xml over WebSockets">>, packet => Text, peer => State#ws_state.peer}), {reply, {text, Text}, State}; @@ -201,7 +202,7 @@ handle_text(Text, #ws_state{ parser = undefined } = State) -> handle_text(Text, #ws_state{parser = Parser} = State) -> case exml_stream:parse(Parser, Text) of {ok, NewParser, Elements} -> - mongoose_metrics:update(global, [data, xmpp, received, raw], byte_size(Text)), + mongoose_metrics:update(global, [data, xmpp, received, c2s, websocket, raw], byte_size(Text)), State1 = State#ws_state{ parser = NewParser }, case maybe_start_fsm(Elements, State1) of {ok, State2} -> From 75bc53e32c44b0b0175744167faa3d31878a6659 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Mon, 20 Feb 2023 06:56:09 +0100 Subject: [PATCH 08/12] extending bosh_SUITE --- big_tests/tests/bosh_SUITE.erl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/big_tests/tests/bosh_SUITE.erl b/big_tests/tests/bosh_SUITE.erl index c2c15f1cab..14cd791125 100644 --- a/big_tests/tests/bosh_SUITE.erl +++ b/big_tests/tests/bosh_SUITE.erl @@ -169,6 +169,13 @@ required_bosh_opts(_Group) -> %%-------------------------------------------------------------------- create_and_terminate_session(Config) -> + MongooseMetrics = [{[global, data, xmpp, received, xml_stanza_size], changed}, + {[global, data, xmpp, sent, xml_stanza_size], changed}, + {[global, data, xmpp, received, c2s, bosh, raw], changed}, + {[global, data, xmpp, sent, c2s, bosh, raw], changed}, + {[global, data, xmpp, received, c2s, tcp, raw], 0}, + {[global, data, xmpp, sent, c2s, tcp, raw], 0}], + PreStoryData = escalus_mongooseim:pre_story([{mongoose_metrics, MongooseMetrics}]), NamedSpecs = escalus_config:get_config(escalus_users, Config), CarolSpec = proplists:get_value(?config(user, Config), NamedSpecs), Conn = escalus_connection:connect(CarolSpec), @@ -188,6 +195,8 @@ create_and_terminate_session(Config) -> Terminate = escalus_bosh:session_termination_body(get_bosh_rid(Conn), Sid), ok = bosh_send_raw(Conn, Terminate), + escalus_mongooseim:post_story(PreStoryData), + %% Assert the session was terminated. wait_for_zero_bosh_sessions(). From 76dd07f5d33cc55c713243437781d8843608b28b Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Mon, 20 Feb 2023 07:34:31 +0100 Subject: [PATCH 09/12] exteding mim_c2s_SUITE --- big_tests/tests/mim_c2s_SUITE.erl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/big_tests/tests/mim_c2s_SUITE.erl b/big_tests/tests/mim_c2s_SUITE.erl index a178b88e6b..ef07062203 100644 --- a/big_tests/tests/mim_c2s_SUITE.erl +++ b/big_tests/tests/mim_c2s_SUITE.erl @@ -64,7 +64,14 @@ end_per_testcase(Name, Config) -> %% tests %%-------------------------------------------------------------------- two_users_can_log_and_chat(Config) -> - escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) -> + MongooseMetrics = [{[global, data, xmpp, received, xml_stanza_size], changed}, + {[global, data, xmpp, sent, xml_stanza_size], changed}, + {[global, data, xmpp, received, c2s, tcp, raw], changed}, + {[global, data, xmpp, sent, c2s, tcp, raw], changed}, + {[global, data, xmpp, received, c2s, tls, raw], 0}, + {[global, data, xmpp, sent, c2s, tls, raw], 0}], + escalus:fresh_story([{mongoose_metrics, MongooseMetrics} | Config], + [{alice, 1}, {bob, 1}], fun(Alice, Bob) -> escalus_client:send(Alice, escalus_stanza:chat_to(Bob, <<"Hi!">>)), escalus:assert(is_chat_message, [<<"Hi!">>], escalus_client:wait_for_stanza(Bob)), escalus_client:send(Bob, escalus_stanza:chat_to(Alice, <<"Hi!">>)), From 7984f3eefbaec75904bf790b30433ef26742dac5 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Mon, 20 Feb 2023 08:03:47 +0100 Subject: [PATCH 10/12] extending connect_SUITE --- big_tests/tests/connect_SUITE.erl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/big_tests/tests/connect_SUITE.erl b/big_tests/tests/connect_SUITE.erl index 1b542608ef..b78d48d679 100644 --- a/big_tests/tests/connect_SUITE.erl +++ b/big_tests/tests/connect_SUITE.erl @@ -87,6 +87,7 @@ tls_groups()-> {group, starttls}, {group, c2s_noproc}, {group, feature_order}, + metrics_test, %% must follow right after feature_order group {group, tls} ]. @@ -346,6 +347,18 @@ verify_features(Conn, Features) -> get_feature(Feature, FeatureList) -> lists:keyfind(Feature, 1, FeatureList). +metrics_test(Config) -> + MongooseMetrics = [{[global, data, xmpp, received, xml_stanza_size], changed}, + {[global, data, xmpp, sent, xml_stanza_size], changed}, + {[global, data, xmpp, received, c2s, tls, raw], changed}, + {[global, data, xmpp, sent, c2s, tls, raw], changed}, + %% TCP traffic before starttls + {[global, data, xmpp, received, c2s, tcp, raw], changed}, + {[global, data, xmpp, sent, c2s, tcp, raw], changed}], + PreStoryData = escalus_mongooseim:pre_story([{mongoose_metrics, MongooseMetrics}]), + tls_authenticate(Config), + escalus_mongooseim:post_story(PreStoryData). + tls_authenticate(Config) -> %% Given UserSpec = escalus_fresh:create_fresh_user(Config, ?SECURE_USER), From fe0e673cb705b9c3539573dfc49699ab53a58671 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Mon, 20 Feb 2023 08:36:24 +0100 Subject: [PATCH 11/12] extending websockets_SUITE --- big_tests/tests/websockets_SUITE.erl | 37 ++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/big_tests/tests/websockets_SUITE.erl b/big_tests/tests/websockets_SUITE.erl index 5cef217279..3ba6140320 100644 --- a/big_tests/tests/websockets_SUITE.erl +++ b/big_tests/tests/websockets_SUITE.erl @@ -27,7 +27,8 @@ -define(REGISTRATION_TIMEOUT, 2). %% seconds all() -> - [{group, ws_chat}, + [metrics_test, + {group, ws_chat}, {group, wss_chat}]. groups() -> @@ -47,23 +48,24 @@ suite() -> %%-------------------------------------------------------------------- init_per_suite(Config) -> - escalus:init_per_suite(Config). + Config1 = escalus:init_per_suite(Config), + escalus:create_users(Config1, escalus:get_users([alice, geralt, geralt_s, carol])). end_per_suite(Config) -> - escalus:end_per_suite(Config). + Config1 = escalus:delete_users(Config, escalus:get_users([alice, geralt, geralt_s, carol])), + escalus:end_per_suite(Config1). init_per_group(GroupName, Config) -> - Config1 = escalus:create_users(Config, escalus:get_users([alice, geralt, geralt_s, carol])), case GroupName of wss_chat -> - [{user, geralt_s} | Config1]; + [{user, geralt_s} | Config]; _ -> - [{user, geralt} | Config1] + [{user, geralt} | Config] end. -end_per_group(_GroupName, Config) -> - escalus:delete_users(Config, escalus:get_users([alice, geralt, geralt_s, carol])). +end_per_group(_GroupName, _Config) -> + ok. init_per_testcase(CaseName, Config) -> escalus:init_per_testcase(CaseName, Config). @@ -75,6 +77,25 @@ end_per_testcase(CaseName, Config) -> %% Message tests %%-------------------------------------------------------------------- +metrics_test(Config) -> + MongooseMetrics = [{[global, data, xmpp, received, xml_stanza_size], changed}, + {[global, data, xmpp, sent, xml_stanza_size], changed}, + {[global, data, xmpp, received, c2s, websocket, raw], changed}, + {[global, data, xmpp, sent, c2s, websocket, raw], changed}, + {[global, data, xmpp, received, c2s, tcp, raw], 0}, + {[global, data, xmpp, sent, c2s, tcp, raw], 0}], + escalus:story([{mongoose_metrics, MongooseMetrics} | Config], + [{geralt, 1}, {geralt_s, 1}], + fun(Geralt, GeraltS) -> + + escalus_client:send(GeraltS, escalus_stanza:chat_to(Geralt, <<"Hi!">>)), + escalus:assert(is_chat_message, [<<"Hi!">>], escalus_client:wait_for_stanza(Geralt)), + + escalus_client:send(Geralt, escalus_stanza:chat_to(GeraltS, <<"Hello!">>)), + escalus:assert(is_chat_message, [<<"Hello!">>], escalus_client:wait_for_stanza(GeraltS)) + + end). + chat_msg(Config) -> escalus:story(Config, [{alice, 1}, {?config(user, Config), 1}, {carol, 1}], fun(Alice, Geralt, Carol) -> From 149d420e4eefede5a1fae579ffecaa3a639cc5cb Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Mon, 20 Feb 2023 21:45:45 +0100 Subject: [PATCH 12/12] fixinng vcard_simple_SUITE:request_search_fields/1 test case --- big_tests/tests/vcard_simple_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/big_tests/tests/vcard_simple_SUITE.erl b/big_tests/tests/vcard_simple_SUITE.erl index 080d1cfa3c..c355995802 100644 --- a/big_tests/tests/vcard_simple_SUITE.erl +++ b/big_tests/tests/vcard_simple_SUITE.erl @@ -456,7 +456,7 @@ configure_mod_vcard(Config) -> ldap_opts() -> LDAPOpts = #{filter => <<"(objectClass=inetOrgPerson)">>, base => <<"ou=Users,dc=esl,dc=com">>, - search_fields => [{"Full Name", "cn"}, {"User", "uid"}], + search_fields => [{<<"Full Name">>, <<"cn">>}, {<<"User">>, <<"uid">>}], vcard_map => [{"FN", "%s", ["cn"]}]}, LDAPOptsWithDefaults = config_parser_helper:config([modules, mod_vcard, ldap], LDAPOpts), config_parser_helper:mod_config(mod_vcard, #{backend => ldap, ldap => LDAPOptsWithDefaults}).