Merge lp:~bloodearnest/charms/precise/squid-reverseproxy/metrics into lp:charms/squid-reverseproxy

[Simon Davy]

Add support for sending cache metrics to statsd

Add support for sending cache metrics to statsd.

Sets up a configurable cron job to gather metrics from squid via snmp and pipe them to statsd.

The snmp names of exported metrics can be set in the 'metrics' config, which has a set of defaults.

Also does a small refactor if jinja templating code as it's used in multiple places now.

[Tom Haddon (mthaddon)]

This looks good to me, and has been tested in a staging environment.

review: Approve

=== modified file '.bzrignore'
--- .bzrignore	2013-08-22 02:22:04 +0000
+++ .bzrignore	2014-04-24 20:39:21 +0000
@@ -6,5 +6,5 @@
 *.key
 lib/*
 *.pyc
-exec.d
+exec.d/*
 build/charm-helpers

=== modified file 'Makefile'
--- Makefile	2013-10-29 17:37:20 +0000
+++ Makefile	2014-04-24 20:39:21 +0000
@@ -9,6 +9,10 @@
 
 build: test lint proof
 
+dev: 
+	mkdir -p exec.d
+	bzr branch lp:~canonical-sysadmins/basenode/trunk exec.d/basenode
+
 revision:
 	@test -f revision || echo 0 > revision
 

=== modified file 'config.yaml'
--- config.yaml	2013-12-13 11:07:25 +0000
+++ config.yaml	2014-04-24 20:39:21 +0000
@@ -76,13 +76,17 @@
   snmp_community:
     type: string
     default: ''
-    description: SNMP community string for monitoring the service.
+    description: SNMP community string for monitoring the service. Required for metrics to be enabled.
   snmp_allowed_ips:
     # would like to use "regex" here, but it doesn't appear to work
     type: string
     default: ''
     description: Single, or json-formatted list of, IP (with optional subnet mask) allowed to query SNMP.
     # validator: '[0-9a-zA-z]{6,32}'
+  snmp_port:
+    type: int
+    default: 3401
+    description: Port for snmp service
   nagios_context:
     default: "juju"
     type: string
@@ -104,10 +108,10 @@
     description: >
       The parameters to pass to the nrpe plugin check_http. String will be formatted with config data
   nagios_service_type:
-     default: "generic"
-     type: string
-     description: >
-       What service this component forms part of, e.g. supermassive-squid-cluster. Used by nrpe.
+    default: "generic"
+    type: string
+    description: >
+      What service this component forms part of, e.g. supermassive-squid-cluster. Used by nrpe.
   package_status:
      default: "install"
      type: "string"
@@ -140,3 +144,55 @@
     default: false
     type: boolean
     description: Enables forward proxying
+  metrics_target:
+    default: ""
+    type: string
+    description: Destination for metrics, format "host:port". If not present and
+      valid, metrics disabled.
+  metrics_scheme:
+    default: "dev.$UNIT.squid.$METRIC"
+    type: string
+    description: |
+      Naming scheme for metrics. Special values $UNIT and $METRIC can be used
+      for more complex schemes, e.g. for suffixes for graphite processing .
+  metrics_sample_interval:
+    default: 5
+    type: int
+    description: Period for metrics cron job to run in minutes
+  metrics:
+    default: |
+      cacheCpuUsage
+      cacheCurrentSwapSize
+      cacheDnsSvcTime.5
+      cacheHttpErrors
+      cacheHttpAllSvcTime.5
+      cacheHttpHitSvcTime.5
+      cacheHttpMissSvcTime.5
+      cacheHttpNhSvcTime.5
+      cacheHttpNmSvcTime.5
+      cacheHttpInKb
+      cacheHttpOutKb
+      cacheMaxResSize
+      cacheMemMaxSize
+      cacheMemUsage
+      cacheNumObjCount
+      cachePeerRtt
+      cacheRequestByteRatio.5
+      cacheRequestHitRatio.5
+      cacheSwapHighWM
+      cacheSwapLowWM
+      cacheSwapMaxSize
+      cacheSysNumReads
+      cacheSysPageFaults
+      cacheSysStorage
+      cacheSysVMsize
+    type: string
+    description: "
+      List of SNMP metrics to be exported. Names should match Squid's SNMP
+      names at http://wiki.squid-cache.org/Features/Snmp#Squid_OIDs.  By
+      default, this charm uses the 5min sampling when averages are used and
+      specifies the .5 measurements explicitly. If you want to use 1m or 60m
+      timings, you should be explicit (.1/.60, and probably change the cron job
+      frequency. Warning: any metric starting with 'cachePeer...' will produce
+      1 metric per configured peer, so can increase the number of metrics
+      rapidly if you have lots of peers."

=== added file 'files/squid_metrics.py'
--- files/squid_metrics.py	1970-01-01 00:00:00 +0000
+++ files/squid_metrics.py	2014-04-24 20:39:21 +0000
@@ -0,0 +1,99 @@
+#!/bin/env python
+import shlex
+import sys
+import subprocess
+import time
+import ConfigParser
+
+config_path = "/usr/local/etc/squid_metrics.ini"
+cmd = "snmpwalk -v1 -Cc -Oq -c %s -m /usr/share/squid3/mib.txt localhost:%s"
+
+
+def clean(s):
+    """Make a string safe for statsd scheme"""
+    return s.replace('.', '-').replace('/', '-')
+
+
+def load_config(path):
+    config = ConfigParser.SafeConfigParser()
+    config.read(config_path)
+    return {
+        'unit': config.get('metrics', 'unit'),
+        'scheme': config.get('metrics', 'scheme'),
+        'port': config.get('metrics', 'port'),
+        'community': config.get('metrics', 'community'),
+        'metrics': config.get('metrics', 'metrics').strip().split(','),
+    }
+
+
+class SNMPMetricFailed(Exception):
+    pass
+
+
+def get_snmp_value(cmd, metric):
+    try:
+        output = subprocess.check_output(cmd + (metric,))
+    except subprocess.CalledProcessError:
+        raise SNMPMetricFailed()
+
+    if 'End of MIB' in output:
+        # could not find metric name in SNMP
+        raise SNMPMetricFailed()
+
+    return output.strip()
+
+
+def get_metrics(config):
+
+    tstamp = int(time.time())
+    scheme = config['scheme'].replace('$UNIT', clean(config['unit']))
+    snmp_cmd = tuple(shlex.split(cmd % (config['community'], config['port'])))
+
+    try:
+        raw_peer_names = get_snmp_value(snmp_cmd, 'cachePeerName')
+    except SNMPMetricFailed:
+        # no peers configured
+        peer_names = {}
+    else:
+        lines = (line for line in raw_peer_names.split('\n') if ' ' in line)
+        peer_names = dict(line[25:].split(' ') for line in lines)
+
+    for metric in config['metrics']:
+        # sadly, have to do one call per metric
+        try:
+            output = get_snmp_value(snmp_cmd, metric)
+        except SNMPMetricFailed:
+            # skip metric
+            # TODO log failure to read metric?
+            continue
+
+        # output is like "SQUID-MIB::<key> <value>"
+        for line in output.split('\n'):
+            # ensure only what we asked, as snmpwalk can be overly generous
+            if metric in line:
+                # strip leading 'SQUID-MIB::'
+                key, value = line[11:].split()
+                # use peer hostname rather than index
+                if 'cachePeer' in key:
+                    key, index = key.split('.')
+                    name = peer_names.get(index, index)
+                    key = clean(key) + '.' + clean(name)
+                else:
+                    # avoid introducing uneeded statsd heirarchy
+                    key = clean(key.replace('.0', ''))
+                if '$METRIC' in scheme:
+                    name = scheme.replace('$METRIC', key)
+                else:
+                    name = scheme + ".%s" % key
+                yield (name, value, tstamp)
+
+
+if __name__ == '__main__':
+
+    if len(sys.argv) > 1:
+        config = load_config(sys.argv[1])
+    else:
+        config = load_config(config_path)
+
+    for metric in get_metrics(config):
+        print "%s %s %s" % metric

=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2014-04-23 10:06:38 +0000
+++ hooks/hooks.py	2014-04-24 20:39:21 +0000
@@ -23,6 +23,7 @@
     related_units,
     open_port,
     close_port,
+    local_unit,
 )
 from charmhelpers.fetch import apt_install
 from charmhelpers.contrib.charmsupport.nrpe import NRPE
@@ -34,6 +35,10 @@
 default_squid3_config = "%s/squid.conf" % default_squid3_config_dir
 default_squid3_config_cache_dir = "/var/run/squid3"
 default_nagios_plugin_dir = "/usr/lib/nagios/plugins"
+metrics_cronjob_path = "/etc/cron.d/squid_metrics"
+metrics_script_path = "/usr/local/bin/squid_metrics.py"
+metrics_config_path = "/usr/local/etc/squid_metrics.ini"
+site_mib_path = "/usr/share/mibs/site"
 Server = collections.namedtuple("Server", "name address port options")
 service_affecting_packages = ['squid3']
 
@@ -208,11 +213,19 @@
     return ''.join(random_chars)
 
 
+def render_template(template_name, vars):
+    # deferred import so install hook can install jinja2
+    from jinja2 import Environment, FileSystemLoader
+    templates_dir = os.path.join(os.environ['CHARM_DIR'], 'templates')
+    template_env = Environment(loader=FileSystemLoader(templates_dir))
+    template = template_env.get_template(template_name)
+    return template.render(vars)
+
+
 #------------------------------------------------------------------------------
 # construct_squid3_config:  Convenience function to write squid.conf
 #------------------------------------------------------------------------------
 def construct_squid3_config():
-    from jinja2 import Environment, FileSystemLoader
     config_data = config_get()
     reverse_sites = get_reverse_sites()
     only_direct = set()
@@ -236,9 +249,6 @@
             'options': [],
             }
 
-    templates_dir = os.path.join(os.environ['CHARM_DIR'], 'templates')
-    template_env = Environment(loader=FileSystemLoader(templates_dir))
-
     config_data['cache_l1'] = int(math.ceil(math.sqrt(
         int(config_data['cache_size_mb']) * 1024 / (
             16 * int(config_data['target_objs_per_dir']) * int(
@@ -261,8 +271,7 @@
         'default_refresh_pattern': default_refresh_pattern,
         'need_localacl_defs': need_localacl_defs,
     }
-    template = template_env.get_template('main_config.template').\
-        render(templ_vars)
+    template = render_template('main_config.template', templ_vars)
     write_squid3_config('\n'.join(
         (l.strip() for l in str(template).splitlines())))
 
@@ -292,6 +301,60 @@
         squid3_config.write(contents)
 
 
+def delete_metrics_cronjob(config_path, cron_path):
+    for path in (config_path, cron_path):
+        try:
+            os.unlink(path)
+        except OSError:
+            pass
+
+
+def write_metrics_cronjob(script_path, config_path, cron_path):
+    config_data = config_get()
+
+    # need the following two configs to be valid
+    metrics_target = config_data['metrics_target'].strip()
+    snmp_community = config_data['snmp_community'].strip()
+    if not snmp_community or not metrics_target or ':' not in metrics_target:
+        log("Required config not found (snmp_community and metrics_target), "
+            "disabling metrics")
+        delete_metrics_cronjob(config_path, cron_path)
+        return
+
+    # check we can install the mibs ok
+    try:
+        apt_install('snmp-mibs-downloader', fatal=True)
+    except subprocess.CalledProcessError:
+        log("Could not install snmp-mibs-downloader package, "
+            "disabling monitoring",
+            level="ERROR")
+        raise
+
+    charm_dir = os.environ['CHARM_DIR']
+    statsd_host, statsd_port = metrics_target.split(':', 1)
+    metrics_list = [s.strip() for s in config_data['metrics'].split()]
+    config_data['metrics_list'] = metrics_list
+    config_data['unit_name'] = local_unit()
+
+    # ensure script installed
+    shutil.copy2('%s/files/squid_metrics.py' % charm_dir, metrics_script_path)
+
+    # write the config
+    with open(config_path, 'w') as config:
+        config.write(
+            render_template("squid_metrics_ini.template", config_data))
+
+    # write the crontab
+    with open(cron_path, 'w') as cronjob:
+        cronjob.write(render_template("metrics_cronjob.template", {
+            'interval': config_data['metrics_sample_interval'],
+            'script': script_path,
+            'config': config_path,
+            'statsd_host': statsd_host,
+            'statsd_port': statsd_port,
+        }))
+
+
 #------------------------------------------------------------------------------
 # service_squid3:  Convenience function to start/stop/restart/reload
 #                   the squid3 service
@@ -378,7 +441,7 @@
 
 
 def install_packages():
-    apt_install("squid3 squidclient python-jinja2".split(), fatal=True)
+    apt_install("squid3 squidclient python-jinja2 snmp".split(), fatal=True)
     ensure_package_status(service_affecting_packages,
                           config_get('package_status'))
 
@@ -387,12 +450,12 @@
 # Hook functions
 ###############################################################################
 def install_hook():
+    charm_dir = os.environ['CHARM_DIR']
     if not os.path.exists(default_squid3_config_dir):
         os.mkdir(default_squid3_config_dir, 0600)
     if not os.path.exists(default_squid3_config_cache_dir):
         os.mkdir(default_squid3_config_cache_dir, 0600)
-    shutil.copy2('%s/files/default.squid3' % (
-        os.environ['CHARM_DIR']), '/etc/default/squid3')
+    shutil.copy2('%s/files/default.squid3' % charm_dir, '/etc/default/squid3')
     install_packages()
     return True
 
@@ -405,6 +468,10 @@
     ensure_package_status(service_affecting_packages,
                           config_get('package_status'))
 
+    write_metrics_cronjob(metrics_script_path,
+                          metrics_config_path,
+                          metrics_cronjob_path)
+
     if service_squid3("check"):
         updated_service_ports = get_service_ports()
         update_service_ports(old_service_ports, updated_service_ports)

=== modified file 'hooks/tests/test_config_changed_hooks.py'
--- hooks/tests/test_config_changed_hooks.py	2013-08-26 21:18:03 +0000
+++ hooks/tests/test_config_changed_hooks.py	2014-04-24 20:39:21 +0000
@@ -21,6 +21,7 @@
         self.notify_cached_website = self.patch_hook("notify_cached_website")
         self.log = self.patch_hook("log")
         self.config_get = self.patch_hook("config_get")
+        self.write_metrics_cronjob = self.patch_hook("write_metrics_cronjob")
 
     def patch_hook(self, hook_name):
         mock_controller = patch.object(hooks, hook_name)

=== added file 'hooks/tests/test_metrics.py'
--- hooks/tests/test_metrics.py	1970-01-01 00:00:00 +0000
+++ hooks/tests/test_metrics.py	2014-04-24 20:39:21 +0000
@@ -0,0 +1,191 @@
+import sys
+import textwrap
+from types import ModuleType
+
+from testtools import TestCase
+from mock import patch, mock_open
+
+import hooks
+
+# import cron script as a standalone module
+with open('files/squid_metrics.py') as script:
+    source = script.read()
+squid_metrics = ModuleType('squid_metrics.py')
+sys.modules['squid_metrics'] = squid_metrics
+exec source in squid_metrics.__dict__
+
+
+class MetricsTestCase(TestCase):
+
+    def add_patch(self, *args, **kwargs):
+        p = patch(*args, **kwargs)
+        self.addCleanup(p.stop)
+        return p.start()
+
+    def setUp(self):
+        super(MetricsTestCase, self).setUp()
+
+        self.open = mock_open()
+        self.add_patch("hooks.open", self.open, create=True)
+        self.copy2 = self.add_patch("shutil.copy2")
+        self.config_get = self.add_patch("hooks.config_get")
+        self.local_unit = self.add_patch("hooks.local_unit")
+        self.log = self.add_patch("hooks.log")
+
+        self.config_get.return_value = {
+            'metrics_sample_interval': 5,
+            'metrics_scheme': 'scheme',
+            'metrics_target': 'localhost:4321',
+            'snmp_port': '1234',
+            'snmp_community': 'community',
+            'metrics': """ a b
+                c """
+        }
+        self.local_unit.return_value = "unit/0"
+
+    @patch('hooks.os.unlink')
+    def test_write_metrics_cronjob_disabled_no_community(self, mock_unlink):
+        self.config_get.return_value['snmp_community'] = ''
+        hooks.write_metrics_cronjob('/script', '/config', '/cron')
+        self.assertEqual(mock_unlink.mock_calls[0][1][0], '/config')
+        self.assertEqual(mock_unlink.mock_calls[1][1][0], '/cron')
+
+    @patch('hooks.os.unlink')
+    def test_write_metrics_cronjob_disabled_no_target(self, mock_unlink):
+        self.config_get.return_value['metrics_target'] = ''
+        hooks.write_metrics_cronjob('/script', '/config', '/cron')
+        self.assertEqual(mock_unlink.mock_calls[0][1][0], '/config')
+        self.assertEqual(mock_unlink.mock_calls[1][1][0], '/cron')
+
+    @patch('hooks.os.unlink')
+    def test_write_metrics_cronjob_disabled_bad_target(self, mock_unlink):
+        self.config_get.return_value['metrics_target'] = 'sadfsadf'
+        hooks.write_metrics_cronjob('/script', '/config', '/cron')
+        self.assertEqual(mock_unlink.mock_calls[0][1][0], '/config')
+        self.assertEqual(mock_unlink.mock_calls[1][1][0], '/cron')
+
+    def test_write_metrics_cronjob_enabled(self):
+        self.config_get.return_value['metrics_target'] = 'localhost:4321'
+
+        hooks.write_metrics_cronjob('/script', '/config', '/cron')
+
+        # first open
+        config_open_args = self.open.mock_calls[0][1]
+        self.assertEqual(config_open_args, ('/config', 'w'))
+
+        config_write = self.open.mock_calls[2][1][0]
+        expected_config = textwrap.dedent("""
+            [metrics]
+            unit: unit/0
+            scheme: scheme
+            port: 1234
+            community: community
+            metrics: a,b,c
+        """).strip()
+        self.assertEqual(config_write, expected_config)
+
+        # 2nd open
+        cron_open_args = self.open.mock_calls[4][1]
+        self.assertEqual(cron_open_args, ('/cron', 'w'))
+
+        cron_write = self.open.mock_calls[6][1][0]
+        expected_cron = textwrap.dedent("""
+           # crontab for pushing squid metrics to statsd
+           */5 * * * * root python /script /config | nc localhost 4321
+        """).strip()
+        self.assertEqual(cron_write, expected_cron)
+
+
+class MetricsJobTestCase(TestCase):
+
+    def add_patch(self, *args, **kwargs):
+        p = patch(*args, **kwargs)
+        self.addCleanup(p.stop)
+        return p.start()
+
+    def setUp(self):
+        super(MetricsJobTestCase, self).setUp()
+
+        self.config = {
+            'unit': 'unit/0',
+            'scheme': 'dev.$UNIT.$METRIC.5min',
+            'port': '1234',
+            'community': 'public',
+            'metrics': [],
+        }
+
+        self.snmp = self.add_patch('squid_metrics.get_snmp_value')
+
+        self.time = self.add_patch('squid_metrics.time.time')
+        self.time.return_value = 1
+
+    def test_periods(self):
+
+        self.config['metrics'] = [
+            'first', 'second', 'third.5', 'forth.60'
+        ]
+
+        self.snmp.side_effect = [
+            squid_metrics.SNMPMetricFailed(),  # no cache peers
+            'SQUID-MIB::first 1',
+            'SQUID-MIB::second.0 2',
+            'SQUID-MIB::third.5 3',
+            'SQUID-MIB::forth.60 4',
+        ]
+
+        metrics = list(squid_metrics.get_metrics(self.config))
+
+        self.assertEqual(metrics, [
+            ('dev.unit-0.first.5min', '1', 1),
+            ('dev.unit-0.second.5min', '2', 1),
+            ('dev.unit-0.third-5.5min', '3', 1),
+            ('dev.unit-0.forth-60.5min', '4', 1),
+        ])
+
+    def test_peers_with_names(self):
+        self.config['metrics'] = [
+            'cachePeerRtt'
+        ]
+
+        self.snmp.side_effect = [
+            (
+                'SQUID-MIB::cachePeerName.1 one.com\n' +
+                'SQUID-MIB::cachePeerName.2 two.org\n' +
+                'SQUID-MIB::cachePeerName.3 three.net\n'
+            ),
+            (
+                'SQUID-MIB::cachePeerRtt.1 1\n' +
+                'SQUID-MIB::cachePeerRtt.2 2\n' +
+                'SQUID-MIB::cachePeerRtt.3 3\n'
+            ),
+        ]
+
+        metrics = list(squid_metrics.get_metrics(self.config))
+
+        self.assertEqual(metrics, [
+            ('dev.unit-0.cachePeerRtt.one-com.5min', '1', 1),
+            ('dev.unit-0.cachePeerRtt.two-org.5min', '2', 1),
+            ('dev.unit-0.cachePeerRtt.three-net.5min', '3', 1),
+        ])
+
+    def test_peers_without_names(self):
+        self.config['metrics'] = [
+            'cachePeerRtt'
+        ]
+
+        self.snmp.side_effect = [
+            squid_metrics.SNMPMetricFailed(),
+            (
+                'SQUID-MIB::cachePeerRtt.1 1\n' +
+                'SQUID-MIB::cachePeerRtt.2 2\n' +
+                'SQUID-MIB::cachePeerRtt.3 3\n'
+            ),
+        ]
+
+        metrics = list(squid_metrics.get_metrics(self.config))
+
+        self.assertEqual(metrics, [
+            ('dev.unit-0.cachePeerRtt.1.5min', '1', 1),
+            ('dev.unit-0.cachePeerRtt.2.5min', '2', 1),
+            ('dev.unit-0.cachePeerRtt.3.5min', '3', 1),
+        ])

=== modified file 'templates/main_config.template'
--- templates/main_config.template	2014-04-23 10:06:38 +0000
+++ templates/main_config.template	2014-04-24 20:39:21 +0000
@@ -21,9 +21,9 @@
 snmp_access allow snmp_access snmp_source
 {% endif -%}
 {% endfor -%}
+snmp_access allow localhost
 snmp_access deny all
-snmp_port 3401
-snmp_incoming_address {{ config.my_ip_address }}
+snmp_port {{ config.snmp_port }}
 {% endif -%}
 
 via {{ config.via }}

=== added file 'templates/metrics_cronjob.template'
--- templates/metrics_cronjob.template	1970-01-01 00:00:00 +0000
+++ templates/metrics_cronjob.template	2014-04-24 20:39:21 +0000
@@ -0,0 +1,3 @@
+# crontab for pushing squid metrics to statsd
+*/{{ interval }} * * * * root python {{ script }} {{ config }} | nc {{ statsd_host }} {{statsd_port }}
+

=== added file 'templates/squid_metrics_ini.template'
--- templates/squid_metrics_ini.template	1970-01-01 00:00:00 +0000
+++ templates/squid_metrics_ini.template	2014-04-24 20:39:21 +0000
@@ -0,0 +1,6 @@
+[metrics]
+unit: {{ unit_name }}
+scheme: {{ metrics_scheme }}
+port: {{ snmp_port }}
+community: {{ snmp_community }}
+metrics: {{ metrics_list|join(',') }}