Ref #3407: Prepare path from Statsd to Prometheus

docs/configuration/settings.rst

@@ -304,18 +304,6 @@ hello view <api-utilities>`.
 Logging and Monitoring
 ======================
 
-+------------------------+----------------------------------------+--------------------------------------------------------------------------+
-| Setting name           | Default                                | What does it do?                                                         |
-+========================+========================================+==========================================================================+
-| kinto.statsd_backend   | ``kinto.core.statsd``                  | The Python **dotted** location of the StatsD module that should be used  |
-|                        |                                        | for monitoring. Useful to plug custom implementations like Datadog™.     |
-+------------------------+----------------------------------------+--------------------------------------------------------------------------+
-| kinto.statsd_prefix    | ``kinto``                              | The prefix to use when sending data to statsd.                           |
-+------------------------+----------------------------------------+--------------------------------------------------------------------------+
-| kinto.statsd_url       | ``None``                               | The fully qualified URL to use to connect to the statsd host. e.g.       |
-|                        |                                        | ``udp://localhost:8125``                                                 |
-+------------------------+----------------------------------------+--------------------------------------------------------------------------+
-
 Standard Logging
 ::::::::::::::::
 
@@ -425,19 +413,47 @@ Or the equivalent environment variables:
     The application sends an event on startup (mainly for setup check).
 
 
+.. _monitoring-with-statsd:
+
 Monitoring with StatsD
 ::::::::::::::::::::::
 
 Requires the ``statsd`` package.
 
-StatsD metrics can be enabled (disabled by default):
++------------------------+----------------------------------------+--------------------------------------------------------------------------+
+| Setting name           | Default                                | What does it do?                                                         |
++========================+========================================+==========================================================================+
+| kinto.statsd_backend   | ``kinto.core.statsd``                  | The Python **dotted** location of the StatsD module that should be used  |
+|                        |                                        | for monitoring. Useful to plug custom implementations like Datadog™.     |
++------------------------+----------------------------------------+--------------------------------------------------------------------------+
+| kinto.statsd_prefix    | ``kinto``                              | The prefix to use when sending data to statsd.                           |
++------------------------+----------------------------------------+--------------------------------------------------------------------------+
+| kinto.statsd_url       | ``None``                               | The fully qualified URL to use to connect to the statsd host. e.g.       |
+|                        |                                        | ``udp://host:8125``                                                      |
++------------------------+----------------------------------------+--------------------------------------------------------------------------+
+
+
+StatsD metrics can be enabled with (disabled by default):
 
 .. code-block:: ini
 
-    kinto.statsd_url = udp://localhost:8125
+    kinto.statsd_url = udp://host:8125
     # kinto.statsd_prefix = kinto-prod
 
 
+StatsD can also be enabled at the *uWSGI* level:
+
+.. code-block:: ini
+
+    [uwsgi]
+
+    # ...
+
+    enable-metrics = true
+    plugin = dogstatsd
+    stats-push = dogstatsd:host:8125,kinto.{{ $deployment }}
+
+
 Monitoring with New Relic
 :::::::::::::::::::::::::
 
@@ -501,6 +517,9 @@ list of Python modules:
 | ``kinto.plugins.quotas``              | It allows to limit storage per collection size, number of records, etc.  |
 |                                       | (:ref:`more details <api-quotas>`).                                      |
 +---------------------------------------+--------------------------------------------------------------------------+
+| ``kinto.plugins.statsd``              | Send metrics about backend duration, authentication, endpoints hits, ..  |
+|                                       | (:ref:`more details <monitoring-with-statsd>`).                          |
++---------------------------------------+--------------------------------------------------------------------------+
 
 
 There are `many available packages`_ in Pyramid ecosystem, and it is straightforward to build one,

kinto/core/__init__.py

@@ -66,8 +66,8 @@
         "kinto.core.initialization.setup_authentication",
         "kinto.core.initialization.setup_backoff",
         "kinto.core.initialization.setup_sentry",
-        "kinto.core.initialization.setup_statsd",
         "kinto.core.initialization.setup_listeners",
+        "kinto.core.initialization.setup_metrics",
         "kinto.core.events.setup_transaction_hook",
     ),
     "event_listeners": "",

kinto/core/initialization.py

@@ -21,7 +21,7 @@
 from pyramid.settings import asbool, aslist
 from pyramid_multiauth import MultiAuthenticationPolicy, MultiAuthPolicySelected
 
-from kinto.core import cache, errors, permission, storage, utils
+from kinto.core import cache, errors, metrics, permission, storage, utils
 from kinto.core.events import ACTIONS, ResourceChanged, ResourceRead
 
 
@@ -334,51 +334,13 @@ def on_app_created(event):
 
 
 def setup_statsd(config):
-    settings = config.get_settings()
-    config.registry.statsd = None
-
-    if settings["statsd_url"]:
-        statsd_mod = settings["statsd_backend"]
-        statsd_mod = config.maybe_dotted(statsd_mod)
-        client = statsd_mod.load_from_config(config)
-
-        config.registry.statsd = client
-
-        client.watch_execution_time(config.registry.cache, prefix="backend")
-        client.watch_execution_time(config.registry.storage, prefix="backend")
-        client.watch_execution_time(config.registry.permission, prefix="backend")
-
-        # Commit so that configured policy can be queried.
-        config.commit()
-        policy = config.registry.queryUtility(IAuthenticationPolicy)
-        if isinstance(policy, MultiAuthenticationPolicy):
-            for name, subpolicy in policy.get_policies():
-                client.watch_execution_time(subpolicy, prefix="authentication", classname=name)
-        else:
-            client.watch_execution_time(policy, prefix="authentication")
-
-        def on_new_response(event):
-            request = event.request
-
-            # Count unique users.
-            user_id = request.prefixed_userid
-            if user_id:
-                # Get rid of colons in metric packet (see #1282).
-                user_id = user_id.replace(":", ".")
-                client.count("users", unique=user_id)
-
-            # Count authentication verifications.
-            if hasattr(request, "authn_type"):
-                client.count(f"authn_type.{request.authn_type}")
-
-            # Count view calls.
-            service = request.current_service
-            if service:
-                client.count(f"view.{service.name}.{request.method}")
-
-        config.add_subscriber(on_new_response, NewResponse)
-
-        return client
+    # It would be pretty rare to find users that have a custom ``kinto.initialization_sequence`` setting.
+    # But just in case, warn that it will be removed in next major.
+    warnings.warn(
+        "``setup_statsd()`` is now deprecated. Use ``kinto.core.initialization.setup_metrics()`` instead.",
+        DeprecationWarning,
+    )
+    setup_metrics(config)
 
 
 def install_middlewares(app, settings):
@@ -466,6 +428,75 @@ def on_new_response(event):
     config.add_subscriber(on_new_response, NewResponse)
 
 
+def setup_metrics(config):
+    settings = config.get_settings()
+
+    # This does not fully respect the Pyramid/ZCA patterns, but the rest of Kinto uses
+    # `registry.storage`, `registry.cache`, etc. Consistency seems more important.
+    config.registry.__class__.metrics = property(
+        lambda reg: reg.queryUtility(metrics.IMetricsService)
+    )
+
+    def deprecated_registry(self):
+        warnings.warn(
+            "``config.registry.statsd`` is now deprecated. Use ``config.registry.metrics`` instead.",
+            DeprecationWarning,
+        )
+        return self.metrics
+
+    config.registry.__class__.statsd = property(deprecated_registry)
+
+    def on_app_created(event):
+        config = event.app
+        metrics_service = config.registry.metrics
+        if not metrics_service:
+            logger.warning("No metrics service registered.")
+            return
+
+        metrics.watch_execution_time(metrics_service, config.registry.cache, prefix="backend")
+        metrics.watch_execution_time(metrics_service, config.registry.storage, prefix="backend")
+        metrics.watch_execution_time(metrics_service, config.registry.permission, prefix="backend")
+
+        policy = config.registry.queryUtility(IAuthenticationPolicy)
+        if isinstance(policy, MultiAuthenticationPolicy):
+            for name, subpolicy in policy.get_policies():
+                metrics.watch_execution_time(
+                    metrics_service, subpolicy, prefix="authentication", classname=name
+                )
+        else:
+            metrics.watch_execution_time(metrics_service, policy, prefix="authentication")
+
+    config.add_subscriber(on_app_created, ApplicationCreated)
+
+    def on_new_response(event):
+        request = event.request
+        metrics_service = config.registry.metrics
+        if not metrics_service:
+            return
+
+        # Count unique users.
+        user_id = request.prefixed_userid
+        if user_id:
+            # Get rid of colons in metric packet (see #1282).
+            user_id = user_id.replace(":", ".")
+            metrics_service.count("users", unique=user_id)
+
+        # Count authentication verifications.
+        if hasattr(request, "authn_type"):
+            metrics_service.count(f"authn_type.{request.authn_type}")
+
+        # Count view calls.
+        service = request.current_service
+        if service:
+            metrics_service.count(f"view.{service.name}.{request.method}")
+
+    config.add_subscriber(on_new_response, NewResponse)
+
+    # While statsd is deprecated, we include its plugin by default for retro-compability.
+    if settings["statsd_url"]:
+        config.include("kinto.plugins.statsd")
+
+
 class EventActionFilter:
     def __init__(self, actions, config):
         actions = ACTIONS.from_string_list(actions)
@@ -518,11 +549,9 @@ def setup_listeners(config):
             listener_mod = config.maybe_dotted(module_value)
             listener = listener_mod.load_from_config(config, prefix)
 
-        # If StatsD is enabled, monitor execution time of listeners.
-        if getattr(config.registry, "statsd", None):
-            statsd_client = config.registry.statsd
-            key = f"listeners.{name}"
-            listener = statsd_client.timer(key)(listener.__call__)
+        wrapped_listener = metrics.listener_with_timer(
+            config, f"listeners.{name}", listener.__call__
+        )
 
         # Optional filter by event action.
         actions_setting = prefix + "actions"
@@ -548,11 +577,11 @@ def setup_listeners(config):
         options = dict(for_actions=actions, for_resources=resource_names)
 
         if ACTIONS.READ in actions:
-            config.add_subscriber(listener, ResourceRead, **options)
+            config.add_subscriber(wrapped_listener, ResourceRead, **options)
             actions = [a for a in actions if a != ACTIONS.READ]
 
         if len(actions) > 0:
-            config.add_subscriber(listener, ResourceChanged, **options)
+            config.add_subscriber(wrapped_listener, ResourceChanged, **options)
 
 
 def load_default_settings(config, default_settings):

kinto/core/metrics.py

@@ -0,0 +1,57 @@
+import types
+
+from zope.interface import Interface
+
+from kinto.core import utils
+
+
+class IMetricsService(Interface):
+    """
+    An interface that defines the metrics service contract.
+    Any class implementing this must provide all its methods.
+    """
+
+    def timer(key):
+        """
+        Watch execution time.
+        """
+
+    def count(key, count=1, unique=None):
+        """
+        Count occurrences. If `unique` is set, overwrites the counter value
+        on each call.
+        """
+
+
+def watch_execution_time(metrics_service, obj, prefix="", classname=None):
+    """
+    Decorate all methods of an object in order to watch their execution time.
+    Metrics will be named `{prefix}.{classname}.{method}`.
+    """
+    classname = classname or utils.classname(obj)
+    members = dir(obj)
+    for name in members:
+        value = getattr(obj, name)
+        is_method = isinstance(value, types.MethodType)
+        if not name.startswith("_") and is_method:
+            statsd_key = f"{prefix}.{classname}.{name}"
+            decorated_method = metrics_service.timer(statsd_key)(value)
+            setattr(obj, name, decorated_method)
+
+
+def listener_with_timer(config, key, func):
+    """
+    Add a timer with the specified `key` on the specified `func`.
+    This is used to avoid evaluating `config.registry.metrics` during setup time
+    to avoid having to deal with initialization order and configuration committing.
+    """
+
+    def wrapped(*args, **kwargs):
+        metrics_service = config.registry.metrics
+        if not metrics_service:
+            return func(*args, **kwargs)
+        # If metrics are enabled, monitor execution time of listeners.
+        with metrics_service.timer(key):
+            return func(*args, **kwargs)
+
+    return wrapped

kinto/core/statsd.py

@@ -1,63 +1 @@
-import types
-from urllib.parse import urlparse
-
-from pyramid.exceptions import ConfigurationError
-
-from kinto.core import utils
-
-
-try:
-    import statsd as statsd_module
-except ImportError:  # pragma: no cover
-    statsd_module = None
-
-
-class Client:
-    def __init__(self, host, port, prefix):
-        self._client = statsd_module.StatsClient(host, port, prefix=prefix)
-
-    def watch_execution_time(self, obj, prefix="", classname=None):
-        classname = classname or utils.classname(obj)
-        members = dir(obj)
-        for name in members:
-            value = getattr(obj, name)
-            is_method = isinstance(value, types.MethodType)
-            if not name.startswith("_") and is_method:
-                statsd_key = f"{prefix}.{classname}.{name}"
-                decorated_method = self.timer(statsd_key)(value)
-                setattr(obj, name, decorated_method)
-
-    def timer(self, key):
-        return self._client.timer(key)
-
-    def count(self, key, count=1, unique=None):
-        if unique is None:
-            return self._client.incr(key, count=count)
-        else:
-            return self._client.set(key, unique)
-
-
-def statsd_count(request, count_key):
-    statsd = request.registry.statsd
-    if statsd:
-        statsd.count(count_key)
-
-
-def load_from_config(config):
-    # If this is called, it means that a ``statsd_url`` was specified in settings.
-    # (see ``kinto.core.initialization``)
-    # Raise a proper error if the ``statsd`` module is not installed.
-    if statsd_module is None:
-        error_msg = "Please install Kinto with monitoring dependencies (e.g. statsd package)"
-        raise ConfigurationError(error_msg)
-
-    settings = config.get_settings()
-    uri = settings["statsd_url"]
-    uri = urlparse(uri)
-
-    if settings["project_name"] != "":
-        prefix = settings["project_name"]
-    else:
-        prefix = settings["statsd_prefix"]
-
-    return Client(uri.hostname, uri.port, prefix)
+from kinto.plugins.statsd import load_from_config  # noqa: F401

kinto/core/testing.py

@@ -8,9 +8,10 @@
 from cornice import errors as cornice_errors
 from pyramid.url import parse_url_overrides
 
-from kinto.core import DEFAULT_SETTINGS, statsd
+from kinto.core import DEFAULT_SETTINGS
 from kinto.core.storage import generators
 from kinto.core.utils import encode64, follow_subrequest, memcache, sqlalchemy
+from kinto.plugins import statsd
 
 
 skip_if_ci = unittest.skipIf("CI" in os.environ, "ci")