From 19aedd18cfeb5530b991ded8d3482ce9e0f87e73 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Mon, 18 Jun 2018 14:46:21 -0400 Subject: [PATCH] Add plugin for varnishd --- README.rst | 12 ++- etc/newrelic/newrelic-plugin-agent.cfg | 5 + newrelic_plugin_agent/plugins/__init__.py | 3 +- newrelic_plugin_agent/plugins/varnishd.py | 111 ++++++++++++++++++++++ 4 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 newrelic_plugin_agent/plugins/varnishd.py diff --git a/README.rst b/README.rst index 917bf8f..83cbaf7 100644 --- a/README.rst +++ b/README.rst @@ -19,6 +19,7 @@ NewRelic platform. Currently supported backend systems are: - Redis - Riak - uWSGI +- Varnishd Base Requirements ----------------- @@ -254,9 +255,13 @@ UWSGI Installation Notes ------------------------ The UWSGI plugin can communicate either over UNIX domain sockets using the path configuration variable or TCP/IP using the host and port variables. Do not include both. -Make sure you have `enabled stats server +Make sure you have `enabled stats server `_ in your uwsgi config. +Varnishd Installation Notes +------------------------ +The Varnishd plugin uses the local ``varnishstat`` command to get information about your instance. Use the ``instance`` configuration variable to define your local varnish instance. The ``newrelic_instance`` can be used to set the NewRelic instance name, else the Varnish instance name is used. Your varnishstat command will be called from your system path, but can be overriden with ``varnishstat`` configuration variable. + Configuration Example --------------------- @@ -407,6 +412,11 @@ Configuration Example port: 8098 #verify_ssl_cert: true + varnishd: + instance: varnish1 + newrelic_instance: my_varnish_server + varnishstat: /usr/local/bin/varnishstat + Daemon: user: newrelic pidfile: /var/run/newrelic/newrelic-plugin-agent.pid diff --git a/etc/newrelic/newrelic-plugin-agent.cfg b/etc/newrelic/newrelic-plugin-agent.cfg index f957d04..cdf3791 100644 --- a/etc/newrelic/newrelic-plugin-agent.cfg +++ b/etc/newrelic/newrelic-plugin-agent.cfg @@ -151,6 +151,11 @@ Application: # port: 1717 # path: /path/to/unix/socket + #varnishd: + # instance: local_varnishd_instance + # newrelic_instance: newrelic_instance_title + # varnishstat: /optional/alternative/path + Daemon: user: newrelic pidfile: /var/run/newrelic/newrelic-plugin-agent.pid diff --git a/newrelic_plugin_agent/plugins/__init__.py b/newrelic_plugin_agent/plugins/__init__.py index 98c5cb9..402e97b 100644 --- a/newrelic_plugin_agent/plugins/__init__.py +++ b/newrelic_plugin_agent/plugins/__init__.py @@ -20,4 +20,5 @@ 'rabbitmq': 'newrelic_plugin_agent.plugins.rabbitmq.RabbitMQ', 'redis': 'newrelic_plugin_agent.plugins.redis.Redis', 'riak': 'newrelic_plugin_agent.plugins.riak.Riak', - 'uwsgi': 'newrelic_plugin_agent.plugins.uwsgi.uWSGI'} + 'uwsgi': 'newrelic_plugin_agent.plugins.uwsgi.uWSGI', + 'varnishd': 'newrelic_plugin_agent.plugins.varnishd.Varnishd'} diff --git a/newrelic_plugin_agent/plugins/varnishd.py b/newrelic_plugin_agent/plugins/varnishd.py new file mode 100644 index 0000000..a7b89d8 --- /dev/null +++ b/newrelic_plugin_agent/plugins/varnishd.py @@ -0,0 +1,111 @@ +""" +varnishd 4.0 +""" +import logging +import platform +import subprocess +import json + +from newrelic_plugin_agent.plugins import base + +LOGGER = logging.getLogger(__name__) + + +class Varnishd(base.Plugin): + + GUID = 'com.meetme.newrelic_varnishd_agent' + + KEYS = ['client_req', + 'backend_fail', + 'cache_miss', + 'cache_hit', + 'threads', + 'threads_created', + 'threads_failed', + 'threads_limited', + 'sess_drop', + 'sess_conn', + 'sess_fail', + 'n_lru_nuked', + 'esi_errors', + 'n_expired'] + + def add_datapoints(self, stats): + """Add all of the data points for a node + + :param dict stats: all of the nodes + + """ + # the default varnish instance name is local hostname + instance_name = self.config.get('instance', platform.node()) + newrelic_name = self.config.get('newrelic_instance', instance_name) + base_name = 'Varnish/%s' % newrelic_name + self.add_derive_value('%s/Requests/received' % base_name, 'client_req', + stats['client_req']) + self.add_derive_value('%s/Backend/failures' % base_name, 'backend_fail', + stats['backend_fail']) + self.add_derive_value('%s/Cache/misses' % base_name, 'cache_miss', + stats['cache_miss']) + self.add_derive_value('%s/Cache/hits' % base_name, 'cache_hit', + stats['cache_hit']) + self.add_derive_value('%s/Threads/total' % base_name, 'threads', + stats['threads']) + self.add_derive_value('%s/Threads/created' % base_name, 'threads_created', + stats['threads_created']) + self.add_derive_value('%s/Threads/failed' % base_name, 'threads_failed', + stats['threads_failed']) + self.add_derive_value('%s/Threads/limited' % base_name, 'threads_limited', + stats['threads_limited']) + self.add_derive_value('%s/Sessions/accepted' % base_name, 'sess_conn', + stats['sess_conn']) + self.add_derive_value('%s/Sessions/failed' % base_name, 'sess_fail', + stats['sess_fail']) + self.add_derive_value('%s/Sessions/dropped' % base_name, 'sess_drop', + stats['sess_drop']) + self.add_derive_value('%s/LRU/nuked' % base_name, 'n_lru_nuked', + stats['n_lru_nuked']) + self.add_derive_value('%s/esi/errors' % base_name, 'esi_errors', + stats['esi_errors']) + self.add_derive_value('%s/Objects/expired' % base_name, 'n_expired', + stats['n_expired']) + + def fetch_data(self): + """Fetch the data from varnish stats command + :rtype: dict + + """ + # the default varnish instance name is local hostname + instance = self.config.get('instance', platform.node()) + varnishstat = self.config.get('varnishstat', 'varnishstat') + try: + p = subprocess.Popen( + [varnishstat, "-1", "-j", "-n", instance], stdout=subprocess.PIPE) + stdout, err = p.communicate() + return json.loads(stdout) + except Exception as error: + LOGGER.error('Subprocess error: %r', error) + return {} + + def parse_metrics(self, data): + """ + filters the appropriate metric types from metric KEYS + :rtype: dict + """ + parsed_data = {} + try: + for metricname in self.KEYS: + jsonmetric = "MAIN." + metricname + metric = data[jsonmetric]['value'] + parsed_data[metricname] = metric + except Exception as error: + LOGGER.error('Metrics parsing error: %r', error) + return parsed_data + + def poll(self): + """Poll subprocess for stats data""" + self.initialize() + data = self.fetch_data() + parsed_data = self.parse_metrics(data) + if parsed_data: + self.add_datapoints(parsed_data) + self.finish()