Pre-build metric prefixes and fetch-or-rebuild when needed #3649
Conversation
This comment was marked as outdated.
This comment was marked as outdated.
Codecov Report
@@ Coverage Diff @@
## master #3649 +/- ##
==========================================
+ Coverage 81.08% 81.11% +0.02%
==========================================
Files 433 433
Lines 31926 31938 +12
==========================================
+ Hits 25886 25905 +19
+ Misses 6040 6033 -7
Continue to review full report at Codecov.
|
Sometimes queries are made with the name already sanitised, like through HTTP, so we want to keep track of such names as well.
NelsonVides
force-pushed
the
prebuild_metrics
branch
from
b76eaf9
to
79742d3
Compare
|
small_tests_24 / small_tests / 79742d3 small_tests_23 / small_tests / 79742d3 dynamic_domains_pgsql_mnesia_23 / pgsql_mnesia / 79742d3 dynamic_domains_pgsql_mnesia_24 / pgsql_mnesia / 79742d3 dynamic_domains_mysql_redis_24 / mysql_redis / 79742d3 dynamic_domains_mssql_mnesia_24 / odbc_mssql_mnesia / 79742d3 disco_and_caps_SUITE:disco_with_caps:user_can_query_friend_resources{error,{{assertion_failed,assert_many,false,[is_roster_set],[],[]},
[{escalus_new_assert,assert_true,2,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_new_assert.erl"},
{line,84}]},
{escalus_story,'-make_all_clients_friends/1-fun-0-',2,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,108}]},
{escalus_utils,'-each_with_index/3-fun-0-',3,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_utils.erl"},
{line,87}]},
{lists,foldl,3,[{file,"lists.erl"},{line,1267}]},
{escalus_utils,'-each_with_index/3-fun-0-',3,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_utils.erl"},
{line,87}]},
{lists,foldl,3,[{file,"lists.erl"},{line,1267}]},
{escalus_utils,distinct_pairs,2,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_utils.erl"},
{line,60}]},
{escalus_story,make_all_clients_friends,1,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,106}]}]}}internal_mnesia_24 / internal_mnesia / 79742d3 ldap_mnesia_24 / ldap_mnesia / 79742d3 ldap_mnesia_23 / ldap_mnesia / 79742d3 pgsql_mnesia_24 / pgsql_mnesia / 79742d3 pgsql_mnesia_23 / pgsql_mnesia / 79742d3 pep_SUITE:pep_tests:unsubscribe_after_presence_unsubscription{error,
{{badmatch,
[{xmlel,<<"message">>,
[{<<"from">>,
<<"alice_unsubscribe_after_presence_unsubscription_1728@localhost">>},
{<<"to">>,
<<"bob_unsubscribe_after_presence_unsubscription_1728@localhost/res1">>},
{<<"type">>,<<"headline">>}],
[{xmlel,<<"event">>,
[{<<"xmlns">>,
<<"http://jabber.org/protocol/pubsub#event">>}],
[{xmlel,<<"items">>,
[{<<"node">>,<<"RZpCdgj0kBO6jogzV+MCPQ==">>}],
[{xmlel,<<"item">>,
[{<<"id">>,<<"salmon">>}],
[{xmlel,<<"entry">>,
[{<<"xmlns">>,
<<"http://www.w3.org/2005/Atom">>}],
[]}]}]}]},
{xmlel,<<"headers">>,
[{<<"xmlns">>,<<"http://jabber.org/protocol/shim">>}],
[]}]}]},
[{pep_SUITE,'-unsubscribe_after_presence_unsubscription/1-fun-0-',2,
[{file,"/home/circleci/project/big_tests/tests/pep_SUITE.erl"},
{line,384}]},
{escalus_story,story,4,
[{file,
"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,72}]},
{test_server,ts_tc,3,[{file,"test_server.erl"},{line,1754}]},
{test_server,run_test_case_eval1,6,
[{file,"test_server.erl"},{line,1263}]},
{test_server,run_test_case_eval,9,
[{file,"test_server.erl"},{line,1195}]}]}}pep_SUITE:pep_tests:unsubscribe_after_presence_unsubscription{error,
{{badmatch,
[{xmlel,<<"message">>,
[{<<"from">>,<<"pubsub.localhost">>},
{<<"to">>,
<<"bob_unsubscribe_after_presence_unsubscription_1741@localhost/res1">>},
{<<"type">>,<<"headline">>}],
[{xmlel,<<"event">>,
[{<<"xmlns">>,
<<"http://jabber.org/protocol/pubsub#event">>}],
[{xmlel,<<"items">>,
[{<<"type">>,<<"headline">>},
{<<"node">>,<<"Q8cf+aw8y46GRK/Ps3nSOg==">>}],
[{xmlel,<<"item">>,
[{<<"id">>,<<"fish">>}],
[{xmlel,<<"entry">>,
[{<<"xmlns">>,
<<"http://www.w3.org/2005/Atom">>}],
[]}]}]}]},
{xmlel,<<"delay">>,
[{<<"xmlns">>,<<"urn:xmpp:delay">>},
{<<"from">>,
<<"alice_unsubscribe_after_presence_unsubscription_1741@localhost/res1">>},
{<<"stamp">>,<<"2022-05-17T11:01:48Z">>}],
[]}]}]},
[{pep_SUITE,'-unsubscribe_after_presence_unsubscription/1-fun-0-',2,
[{file,"/home/circleci/project/big_tests/tests/pep_SUITE.erl"},
{line,384}]},
{escalus_story,story,4,
[{file,
"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,72}]},
{test_server,ts_tc,3,[{file,"test_server.erl"},{line,1754}]},
{test_server,run_test_case_eval1,6,
[{file,"test_server.erl"},{line,1263}]},
{test_server,run_test_case_eval,9,
[{file,"test...elasticsearch_and_cassandra_24 / elasticsearch_and_cassandra_mnesia / 79742d3 mssql_mnesia_24 / odbc_mssql_mnesia / 79742d3 riak_mnesia_24 / riak_mnesia / 79742d3 mysql_redis_24 / mysql_redis / 79742d3 pgsql_mnesia_23 / pgsql_mnesia / 79742d3 pep_SUITE:pep_tests:unsubscribe_after_presence_unsubscription{error,
{{badmatch,
[{xmlel,<<"message">>,
[{<<"from">>,
<<"alice_unsubscribe_after_presence_unsubscription_1737@localhost">>},
{<<"to">>,
<<"bob_unsubscribe_after_presence_unsubscription_1737@localhost/res1">>},
{<<"type">>,<<"headline">>}],
[{xmlel,<<"event">>,
[{<<"xmlns">>,
<<"http://jabber.org/protocol/pubsub#event">>}],
[{xmlel,<<"items">>,
[{<<"node">>,<<"20ZOgVTS2neXhBlW5I5WmA==">>}],
[{xmlel,<<"item">>,
[{<<"id">>,<<"salmon">>}],
[{xmlel,<<"entry">>,
[{<<"xmlns">>,
<<"http://www.w3.org/2005/Atom">>}],
[]}]}]}]},
{xmlel,<<"headers">>,
[{<<"xmlns">>,<<"http://jabber.org/protocol/shim">>}],
[]}]}]},
[{pep_SUITE,'-unsubscribe_after_presence_unsubscription/1-fun-0-',2,
[{file,"/home/circleci/project/big_tests/tests/pep_SUITE.erl"},
{line,384}]},
{escalus_story,story,4,
[{file,
"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,72}]},
{test_server,ts_tc,3,[{file,"test_server.erl"},{line,1754}]},
{test_server,run_test_case_eval1,6,
[{file,"test_server.erl"},{line,1263}]},
{test_server,run_test_case_eval,9,
[{file,"test_server.erl"},{line,1195}]}]}} |
src/metrics/mongoose_metrics.erl
Outdated
| -spec all_metrics_are_global() -> boolean(). | ||
| all_metrics_are_global() -> | ||
| mongoose_config:get_opt(all_metrics_are_global). | ||
|
|
||
| get_host_type_prefix(HostType) when is_atom(HostType) -> |
There was a problem hiding this comment.
Is this atom always global? If so, maybe we could match on it explicitly and add a type spec with mongooseim:host_type_or_global()? It would help with type checking and understanding the code.
There was a problem hiding this comment.
Good point, I think it is always global indeed, will double-check and try to fix 👍🏽
src/metrics/mongoose_metrics.erl
Outdated
| @@ -48,6 +48,7 @@ | |||
| remove_host_type_metrics/1, get_report_interval/0, | |||
| sample_metric/1]). | |||
|
|
|||
| -define(PREFIXES, {?MODULE, prefixes}). | |||
There was a problem hiding this comment.
I think an atom mongoose_metrics_prefixes would be marginally faster but still faster.
Unless to have a tuple is some kind of convention to follow.
There was a problem hiding this comment.
Yeah, might be faster, but for once I didn't think about that, I kinda assumed from somewhere we have that convention 🤷🏽 🤔
|
small_tests_24 / small_tests / d9c72a3 small_tests_23 / small_tests / d9c72a3 dynamic_domains_pgsql_mnesia_24 / pgsql_mnesia / d9c72a3 dynamic_domains_mysql_redis_24 / mysql_redis / d9c72a3 ldap_mnesia_24 / ldap_mnesia / d9c72a3 ldap_mnesia_23 / ldap_mnesia / d9c72a3 internal_mnesia_24 / internal_mnesia / d9c72a3 dynamic_domains_mssql_mnesia_24 / odbc_mssql_mnesia / d9c72a3 elasticsearch_and_cassandra_24 / elasticsearch_and_cassandra_mnesia / d9c72a3 pgsql_mnesia_23 / pgsql_mnesia / d9c72a3 mysql_redis_24 / mysql_redis / d9c72a3 pgsql_mnesia_24 / pgsql_mnesia / d9c72a3 mssql_mnesia_24 / odbc_mssql_mnesia / d9c72a3 riak_mnesia_24 / riak_mnesia / d9c72a3 |
Use a smaller key for the persistent_terms map, and remove unreachable code.
|
small_tests_24 / small_tests / 2bf64f6 small_tests_23 / small_tests / 2bf64f6 dynamic_domains_pgsql_mnesia_24 / pgsql_mnesia / 2bf64f6 dynamic_domains_pgsql_mnesia_23 / pgsql_mnesia / 2bf64f6 dynamic_domains_mysql_redis_24 / mysql_redis / 2bf64f6 ldap_mnesia_24 / ldap_mnesia / 2bf64f6 ldap_mnesia_23 / ldap_mnesia / 2bf64f6 internal_mnesia_24 / internal_mnesia / 2bf64f6 dynamic_domains_mssql_mnesia_24 / odbc_mssql_mnesia / 2bf64f6 muc_SUITE:hibernation:hibernated_room_can_be_queried_for_archive{error,{{assertion_failed,assert,is_groupchat_message,
[<<"Restorable message">>],
undefined,"undefined"},
[{escalus_new_assert,assert_true,2,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_new_assert.erl"},
{line,84}]},
{muc_SUITE,wait_for_mam_result,3,
[{file,"/home/circleci/project/big_tests/tests/muc_SUITE.erl"},
{line,4383}]},
{muc_SUITE,'-hibernated_room_can_be_queried_for_archive/1-fun-0-',3,
[{file,"/home/circleci/project/big_tests/tests/muc_SUITE.erl"},
{line,4124}]},
{escalus_story,story,4,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,72}]},
{muc_SUITE,hibernated_room_can_be_queried_for_archive,1,
[{file,"/home/circleci/project/big_tests/tests/muc_SUITE.erl"},
{line,4120}]},
{test_server,ts_tc,3,[{file,"test_server.erl"},{line,1783}]},
{test_server,run_test_case_eval1,6,
[{file,"test_server.erl"},{line,1292}]},
{test_server,run_test_case_eval,9,
[{file,"test_server.erl"},{line,1224}]}]}}smart_markers_SUITE:regular:one2one:marker_for_thread_can_be_fetched{error,
{{fetch_marker,ok,
[{times,50,
{error,
{badmatch,[]},
[{smart_markers_SUITE,'-verify_marker_fetch/4-fun-6-',3,
[{file,
"/home/circleci/project/big_tests/tests/smart_markers_SUITE.erl"},
{line,380}]},
{mongoose_helper,do_wait_until,2,
[{file,
"/home/circleci/project/big_tests/tests/mongoose_helper.erl"},
{line,374}]},
{escalus_story,story,4,
[{file,
"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,72}]},
{test_server,ts_tc,3,
[{file,"test_server.erl"},{line,1783}]},
{test_server,run_test_case_eval1,6,
[{file,"test_server.erl"},{line,1292}]},
{test_server,run_test_case_eval,9,
[{file,"test_server.erl"},{line,1224}]}]}}]},
[{mongoose_helper,do_wait_until,2,
[{file,"/home/circleci/project/big_tests/tests/mongoose_helper.erl"},
{line,371}]},
{escalus_story,story,4,
[{file,
"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,72}]},
{test_server,ts_tc,3,[{file,"test_server.erl"},{line,1783}]},
{test_server,run_test_case_eval1,6,
[{file,"test_server.erl"},{line,1292}]},
{test_server,run_test_case_eval,9,
[{file,"test_server.erl"},{line,1224}]}]}}elasticsearch_and_cassandra_24 / elasticsearch_and_cassandra_mnesia / 2bf64f6 pgsql_mnesia_23 / pgsql_mnesia / 2bf64f6 pgsql_mnesia_24 / pgsql_mnesia / 2bf64f6 mysql_redis_24 / mysql_redis / 2bf64f6 riak_mnesia_24 / riak_mnesia / 2bf64f6 mssql_mnesia_24 / odbc_mssql_mnesia / 2bf64f6 muc_SUITE:hibernation:hibernated_room_can_be_queried_for_archive{error,{{assertion_failed,assert,is_groupchat_message,
[<<"Restorable message">>],
undefined,"undefined"},
[{escalus_new_assert,assert_true,2,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_new_assert.erl"},
{line,84}]},
{muc_SUITE,wait_for_mam_result,3,
[{file,"/home/circleci/project/big_tests/tests/muc_SUITE.erl"},
{line,4383}]},
{muc_SUITE,'-hibernated_room_can_be_queried_for_archive/1-fun-0-',3,
[{file,"/home/circleci/project/big_tests/tests/muc_SUITE.erl"},
{line,4124}]},
{escalus_story,story,4,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,72}]},
{muc_SUITE,hibernated_room_can_be_queried_for_archive,1,
[{file,"/home/circleci/project/big_tests/tests/muc_SUITE.erl"},
{line,4120}]},
{test_server,ts_tc,3,[{file,"test_server.erl"},{line,1783}]},
{test_server,run_test_case_eval1,6,
[{file,"test_server.erl"},{line,1292}]},
{test_server,run_test_case_eval,9,
[{file,"test_server.erl"},{line,1224}]}]}}pep_SUITE:pep_tests:unsubscribe_after_presence_unsubscription{error,
{{badmatch,
[{xmlel,<<"message">>,
[{<<"from">>,
<<"alice_unsubscribe_after_presence_unsubscription_1963@localhost">>},
{<<"to">>,
<<"bob_unsubscribe_after_presence_unsubscription_1963@localhost/res1">>},
{<<"type">>,<<"headline">>}],
[{xmlel,<<"event">>,
[{<<"xmlns">>,
<<"http://jabber.org/protocol/pubsub#event">>}],
[{xmlel,<<"items">>,
[{<<"node">>,<<"NQx87NLRPVjetpFkIbexdw==">>}],
[{xmlel,<<"item">>,
[{<<"id">>,<<"salmon">>}],
[{xmlel,<<"entry">>,
[{<<"xmlns">>,
<<"http://www.w3.org/2005/Atom">>}],
[]}]}]}]},
{xmlel,<<"headers">>,
[{<<"xmlns">>,<<"http://jabber.org/protocol/shim">>}],
[]}]}]},
[{pep_SUITE,'-unsubscribe_after_presence_unsubscription/1-fun-0-',2,
[{file,"/home/circleci/project/big_tests/tests/pep_SUITE.erl"},
{line,384}]},
{escalus_story,story,4,
[{file,
"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,72}]},
{test_server,ts_tc,3,[{file,"test_server.erl"},{line,1783}]},
{test_server,run_test_case_eval1,6,
[{file,"test_server.erl"},{line,1292}]},
{test_server,run_test_case_eval,9,
[{file,"test_server.erl"},{line,1224}]}]}}rest_client_SUITE:roster:add_contact_and_invite{error,
{timeout_when_waiting_for_stanza,
[{escalus_client,wait_for_stanza,
[{client,<<"bob_add_contact_and_invite_2354@localhost/res1">>,
escalus_tcp,<0.9355.2>,
[{event_manager,<0.9329.2>},
{server,<<"localhost">>},
{username,<<"bOb_add_contact_and_invite_2354">>},
{resource,<<"res1">>}],
[{event_client,
[{event_manager,<0.9329.2>},
{server,<<"localhost">>},
{username,<<"bOb_add_contact_and_invite_2354">>},
{resource,<<"res1">>}]},
{resource,<<"res1">>},
{username,<<"bob_add_contact_and_invite_2354">>},
{server,<<"localhost">>},
{host,<<"localhost">>},
{port,5222},
{auth,{escalus_auth,auth_plain}},
{wspath,undefined},
{username,<<"bOb_add_contact_and_invite_2354">>},
{server,<<"localhost">>},
{password,<<"makrolika">>},
{stream_id,<<"35179fb63ee7e790">>}]},
5000],
[{file,
"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_client.erl"},
{line,136}]},
{rest_client_SUITE,add_contact_check_roster_push,2,
[{file,
"/home/circleci/project/big_tests/tests/rest_client_SUITE.erl"},
{line,1307}]},
{rest_client_SUITE,'-add_contact_and_invite/1-fun-0-',2,
[{file,
"/home/circleci/project/big_tests/tests/rest_client_SUITE.erl"},
{line...dynamic_domains_mssql_mnesia_24 / odbc_mssql_mnesia / 2bf64f6 muc_SUITE:hibernation:hibernated_room_can_be_queried_for_archive{error,{{assertion_failed,assert,is_groupchat_message,
[<<"Restorable message">>],
undefined,"undefined"},
[{escalus_new_assert,assert_true,2,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_new_assert.erl"},
{line,84}]},
{muc_SUITE,wait_for_mam_result,3,
[{file,"/home/circleci/project/big_tests/tests/muc_SUITE.erl"},
{line,4383}]},
{muc_SUITE,'-hibernated_room_can_be_queried_for_archive/1-fun-0-',3,
[{file,"/home/circleci/project/big_tests/tests/muc_SUITE.erl"},
{line,4124}]},
{escalus_story,story,4,
[{file,"/home/circleci/project/big_tests/_build/default/lib/escalus/src/escalus_story.erl"},
{line,72}]},
{muc_SUITE,hibernated_room_can_be_queried_for_archive,1,
[{file,"/home/circleci/project/big_tests/tests/muc_SUITE.erl"},
{line,4120}]},
{test_server,ts_tc,3,[{file,"test_server.erl"},{line,1783}]},
{test_server,run_test_case_eval1,6,
[{file,"test_server.erl"},{line,1292}]},
{test_server,run_test_case_eval,9,
[{file,"test_server.erl"},{line,1224}]}]}} |
This PR addresses the fact that metric prefixes are sanitised every single time a metric is going to be updated, which happens potentially hundreds of times per message, turning into an important waste. Here we instead build all host-type prefixes early on and store them on a persistent term, and fetch these values when updating the metrics. If a prefix wasn't built at that point, we can only then rebuild it.