Merge lp:~bloodearnest/charms/precise/squid-reverseproxy/fix-stats-cron into lp:charms/squid-reverseproxy

[Simon Davy]

fix statsd wire format and send directly in metrics script

fix statsd wire format and send directly in metrics script

[Tom Haddon (mthaddon)]

Looks good to me, and tested to work on a staging environment

review: Approve

=== modified file 'files/squid_metrics.py'
--- files/squid_metrics.py	2014-03-28 17:05:31 +0000
+++ files/squid_metrics.py	2014-05-02 14:57:34 +0000
@@ -1,10 +1,122 @@
 #!/bin/env python
 import shlex
 import sys
+import socket
 import subprocess
-import time
 import ConfigParser
 
+
+METRIC_TYPES = {
+    'cacheSysVMsize': 'g',  # Storage Mem size in KB
+    'cacheSysStorage': 'g',  # Storage Swap size in KB
+    'cacheUptime': 'c',  # The Uptime of the cache in timeticks
+    'cacheMemMaxSize': 'g',  # The value of the cache_mem parameter in MB
+    'cacheSwapMaxSize': 'g',  # The total of the cache_dir space allocated in MB
+    'cacheSwapHighWM': 'g',  # Cache Swap High Water Mark
+    'cacheSwapLowWM': 'g',  # Cache Swap Low Water Mark
+    'cacheUniqName': 'g',  # Cache unique host name
+    'cacheSysPageFaults': 'c',  # Page faults with physical i/o
+    'cacheSysNumReads': 'c',  # HTTP I/O number of reads
+    'cacheMemUsage': 'g',  # Total memory accounted for KB
+    'cacheCpuTime': 'c',  # Amount of cpu seconds consumed
+    'cacheCpuUsage': 'g',  # The percentage use of the CPU
+    'cacheMaxResSize': 'g',  # Maximum Resident Size in KB
+    'cacheNumObjCount': 'g',  # Number of objects stored by the cache
+    'cacheCurrentLRUExpiration': 'g',  # Storage LRU Expiration Age
+    'cacheCurrentUnlinkRequests': 'g',  # Requests given to unlinkd
+    'cacheCurrentUnusedFDescrCnt': 'g',  # Available number of file descriptors
+    'cacheCurrentResFileDescrCnt': 'g',  # Reserved number of file descriptors
+    'cacheCurrentFileDescrCnt': 'g',  # Number of file descriptors in use
+    'cacheCurrentFileDescrMax': 'g',  # Highest file descriptors in use
+    'cacheProtoClientHttpRequests': 'c',  # Number of HTTP requests received
+    'cacheHttpHits': 'c',  # Number of HTTP Hits sent to clients from cache
+    'cacheHttpErrors': 'c',  # Number of HTTP Errors sent to clients
+    'cacheHttpInKb': 'c',  # Number of HTTP KB's received from clients
+    'cacheHttpOutKb': 'c',  # Number of HTTP KB's sent to clients
+    'cacheIcpPktsSent': 'c',  # Number of ICP messages sent
+    'cacheIcpPktsRecv': 'c',  # Number of ICP messages received
+    'cacheIcpKbSent': 'c',  # Number of ICP KB's transmitted
+    'cacheIcpKbRecv': 'c',  # Number of ICP KB's received
+    'cacheServerRequests': 'c',  # All requests from the client for the cache server
+    'cacheServerErrors': 'c',  # All errors for the cache server from client requests
+    'cacheServerInKb': 'c',  # KB's of traffic received from servers
+    'cacheServerOutKb': 'c',  # KB's of traffic sent to servers
+    'cacheCurrentSwapSize': 'g',  # Storage Swap size
+    'cacheClients': 'g',  # Number of clients accessing cache
+    'cacheMedianTime.1': 'g',  # The value used to index the table 1/5/60
+    'cacheMedianTime.5': 'g',  #
+    'cacheMedianTime.60': 'g',  #
+    'cacheHttpAllSvcTime.1': 'g',  # HTTP all service time
+    'cacheHttpAllSvcTime.5': 'g',  #
+    'cacheHttpAllSvcTime.60': 'g',  #
+    'cacheHttpMissSvcTime.1': 'g',  # HTTP miss service time
+    'cacheHttpMissSvcTime.5': 'g',  #
+    'cacheHttpMissSvcTime.60': 'g',  #
+    'cacheHttpNmSvcTime.1': 'g',  # HTTP hit not-modified service time
+    'cacheHttpNmSvcTime.5': 'g',  #
+    'cacheHttpNmSvcTime.60': 'g',  #
+    'cacheHttpHitSvcTime.1': 'g',  # HTTP hit service time
+    'cacheHttpHitSvcTime.5': 'g',  #
+    'cacheHttpHitSvcTime.60': 'g',  #
+    'cacheIcpQuerySvcTime.1': 'g',  # ICP query service time
+    'cacheIcpQuerySvcTime.5': 'g',  #
+    'cacheIcpQuerySvcTime.60': 'g',  #
+    'cacheIcpReplySvcTime.1': 'g',  # ICP reply service time
+    'cacheIcpReplySvcTime.5': 'g',  #
+    'cacheIcpReplySvcTime.60': 'g',  #
+    'cacheDnsSvcTime.1': 'g',  # DNS service time
+    'cacheDnsSvcTime.5': 'g',  #
+    'cacheDnsSvcTime.60': 'g',  #
+    'cacheRequestHitRati.1': 'g',  # Request Hit Ratios
+    'cacheRequestHitRati.5': 'g',  #
+    'cacheRequestHitRati.60': 'g',  #
+    'cacheRequestByteRati.1': 'g',  # Byte Hit Ratios
+    'cacheRequestByteRati.5': 'g',  #
+    'cacheRequestByteRati.60': 'g',  #
+    'cacheHttpNhSvcTime.1': 'g',  # HTTP refresh hit service time
+    'cacheHttpNhSvcTime.5': 'g',  #
+    'cacheHttpNhSvcTime.60': 'g',  #
+    'cacheIpEntries': 'g',  # IP Cache Entries
+    'cacheIpRequests': 'c',  # Number of IP Cache requests
+    'cacheIpHits': 'c',  # Number of IP Cache hits
+    'cacheIpPendingHits': 'g',  # Number of IP Cache pending hits
+    'cacheIpNegativeHits': 'c',  # Number of IP Cache pending hits
+    'cacheIpMisses': 'c',  # Number of IP Cache misses
+    'cacheBlockingGetHostByName': 'c',  # Number of blocking gethostbyname requests
+    'cacheAttemptReleaseLckEntries': 'c',  # Number of attempts to release locked IP
+    'cacheFqdnEntries': 'g',  # FQDN Cache entries
+    'cacheFqdnRequests': 'c',  # Number of FQDN Cache requests
+    'cacheFqdnHits': 'c',  # Number of FQDN Cache hits
+    'cacheFqdnPendingHits': 'g',  # Number of FQDN Cache pending hits
+    'cacheFqdnNegativeHits': 'c',  # Number of FQDN Cache negative hits
+    'cacheFqdnMisses': 'c',  # Number of FQDN Cache misses
+    'cacheBlockingGetHostByAddr': 'c',  # Number of blocking gethostbyaddr requests
+    'cacheDnsRequests': 'c',  # Number of external dnsserver requests
+    'cacheDnsReplies': 'c',  # Number of external dnsserver replies
+    'cacheDnsNumberServers': 'c',  # Number of external dnsserver processes
+    'cachePeerIndex': 'g',  # A unique value, greater than zero for each cache_peer instance in the managed system.
+    'cachePeerPortHttp': 'g',  # The port the peer listens for HTTP requests
+    'cachePeerPortIcp': 'g',  # The port the peer listens for ICP requests should be 0 if not configured to send ICP requests
+    'cachePeerType': 'g',  # Peer Type
+    'cachePeerState': 'g',  # The operational state of this peer
+    'cachePeerPingsSent': 'c',  # Number of pings sent to peer
+    'cachePeerPingsAcked': 'c',  # Number of pings received from peer
+    'cachePeerFetches': 'c',  # Number of times this peer was selected
+    'cachePeerRtt': 'g',  # Last known round-trip time to the peer (in ms)
+    'cachePeerIgnored': 'c',  # How many times this peer was ignored
+    'cachePeerKeepAlSent': 'c',  # Number of keepalives sent
+    'cachePeerKeepAlRecv': 'c',  # Number of keepalives received
+    'cacheClientHttpRequests': 'c',  # Number of HTTP requests received from client
+    'cacheClientHttpKb': 'c',  # Amount of total HTTP traffic to this client
+    'cacheClientHttpHits': 'c',  # Number of hits in response to this client's HTTP requests
+    'cacheClientHTTPHitKb': 'c',  # Amount of HTTP hit traffic in KB
+    'cacheClientIcpRequests': 'c',  # Number of ICP requests received from client
+    'cacheClientIcpKb': 'c',  # Amount of total ICP traffic to this client (child)
+    'cacheClientIcpHits': 'c',  # Number of hits in response to this client's ICP requests
+    'cacheClientIcpHitKb': 'c',  # Amount of ICP hit traffic in KB
+}
+
+
 config_path = "/usr/local/etc/squid_metrics.ini"
 cmd = "snmpwalk -v1 -Cc -Oq -c %s -m /usr/share/squid3/mib.txt localhost:%s"
 
@@ -20,7 +132,8 @@
     return {
         'unit': config.get('metrics', 'unit'),
         'scheme': config.get('metrics', 'scheme'),
-        'port': config.get('metrics', 'port'),
+        'snmp_port': config.get('metrics', 'snmp_port'),
+        'statsd_hostport': config.get('metrics', 'statsd_hostport'),
         'community': config.get('metrics', 'community'),
         'metrics': config.get('metrics', 'metrics').strip().split(','),
     }
@@ -45,9 +158,9 @@
 
 def get_metrics(config):
 
-    tstamp = int(time.time())
     scheme = config['scheme'].replace('$UNIT', clean(config['unit']))
-    snmp_cmd = tuple(shlex.split(cmd % (config['community'], config['port'])))
+    snmp_cmd = tuple(
+        shlex.split(cmd % (config['community'], config['snmp_port'])))
 
     try:
         raw_peer_names = get_snmp_value(snmp_cmd, 'cachePeerName')
@@ -59,6 +172,11 @@
         peer_names = dict(line[25:].split(' ') for line in lines)
 
     for metric in config['metrics']:
+
+        if metric not in METRIC_TYPES:
+            # TODO log failure?
+            continue
+
         # sadly, have to do one call per metric
         try:
             output = get_snmp_value(snmp_cmd, metric)
@@ -67,6 +185,8 @@
             # TODO log failure to read metric?
             continue
 
+        metric_type = METRIC_TYPES[metric]
+
         # output is like "SQUID-MIB::<key> <value>"
         for line in output.split('\n'):
             # ensure only what we asked, as snmpwalk can be overly generous
@@ -85,15 +205,22 @@
                     name = scheme.replace('$METRIC', key)
                 else:
                     name = scheme + ".%s" % key
-                yield (name, value, tstamp)
+                yield (name, value, metric_type)
 
 
 if __name__ == '__main__':
 
-    if len(sys.argv) > 1:
-        config = load_config(sys.argv[1])
-    else:
-        config = load_config(config_path)
+    config = load_config(config_path)
+
+    def send(x):
+        print x
+
+    if len(sys.argv) > 1 and sys.argv[1] == "send":
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        host, port = config['statsd_hostport'].split(':')
+        sock.connect((host, int(port)))
+        def send(x):
+            sock.send(x)
 
     for metric in get_metrics(config):
-        print "%s %s %s" % metric
+        send(("%s:%s|%s" % metric).encode('utf8'))

=== modified file 'hooks/tests/test_metrics.py'
--- hooks/tests/test_metrics.py	2014-03-28 17:05:31 +0000
+++ hooks/tests/test_metrics.py	2014-05-02 14:57:34 +0000
@@ -31,6 +31,7 @@
         self.config_get = self.add_patch("hooks.config_get")
         self.local_unit = self.add_patch("hooks.local_unit")
         self.log = self.add_patch("hooks.log")
+        self.apt_install = self.add_patch("hooks.apt_install")
 
         self.config_get.return_value = {
             'metrics_sample_interval': 5,
@@ -78,11 +79,12 @@
             [metrics]
             unit: unit/0
             scheme: scheme
-            port: 1234
+            snmp_port: 1234
+            statsd_hostport: localhost:4321
             community: community
             metrics: a,b,c
         """).strip()
-        self.assertEqual(config_write, expected_config)
+        self.assertEqual(expected_config, config_write)
 
         # 2nd open
         cron_open_args = self.open.mock_calls[4][1]
@@ -91,9 +93,9 @@
         cron_write = self.open.mock_calls[6][1][0]
         expected_cron = textwrap.dedent("""
            # crontab for pushing squid metrics to statsd
-           */5 * * * * root python /script /config | nc localhost 4321
-        """).strip()
-        self.assertEqual(cron_write, expected_cron)
+           */5 * * * * root python /script send
+        """).lstrip()
+        self.assertEqual(expected_cron, cron_write)
 
 
 class MetricsJobTestCase(TestCase):
@@ -109,20 +111,27 @@
         self.config = {
             'unit': 'unit/0',
             'scheme': 'dev.$UNIT.$METRIC.5min',
-            'port': '1234',
+            'snmp_port': '1234',
+            'statsd_hostport': '127.0.0.1:8000',
             'community': 'public',
             'metrics': [],
         }
 
         self.snmp = self.add_patch('squid_metrics.get_snmp_value')
-
-        self.time = self.add_patch('squid_metrics.time.time')
-        self.time.return_value = 1
+        patcher = patch.dict(squid_metrics.METRIC_TYPES, {
+            'first': 'c',
+            'second': 'c',
+            'third.5': 'g',
+            'fourth.60': 'g',
+            'cachePeerRtt': 'g',
+        })
+        patcher.start()
+        self.addCleanup(patcher.stop)
 
     def test_periods(self):
 
         self.config['metrics'] = [
-            'first', 'second', 'third.5', 'forth.60'
+            'first', 'second', 'third.5', 'fourth.60'
         ]
 
         self.snmp.side_effect = [
@@ -130,16 +139,16 @@
             'SQUID-MIB::first 1',
             'SQUID-MIB::second.0 2',
             'SQUID-MIB::third.5 3',
-            'SQUID-MIB::forth.60 4',
+            'SQUID-MIB::fourth.60 4',
         ]
 
         metrics = list(squid_metrics.get_metrics(self.config))
 
         self.assertEqual(metrics, [
-            ('dev.unit-0.first.5min', '1', 1),
-            ('dev.unit-0.second.5min', '2', 1),
-            ('dev.unit-0.third-5.5min', '3', 1),
-            ('dev.unit-0.forth-60.5min', '4', 1),
+            ('dev.unit-0.first.5min', '1', 'c'),
+            ('dev.unit-0.second.5min', '2', 'c'),
+            ('dev.unit-0.third-5.5min', '3', 'g'),
+            ('dev.unit-0.fourth-60.5min', '4', 'g'),
         ])
 
     def test_peers_with_names(self):
@@ -163,9 +172,9 @@
         metrics = list(squid_metrics.get_metrics(self.config))
 
         self.assertEqual(metrics, [
-            ('dev.unit-0.cachePeerRtt.one-com.5min', '1', 1),
-            ('dev.unit-0.cachePeerRtt.two-org.5min', '2', 1),
-            ('dev.unit-0.cachePeerRtt.three-net.5min', '3', 1),
+            ('dev.unit-0.cachePeerRtt.one-com.5min', '1', 'g'),
+            ('dev.unit-0.cachePeerRtt.two-org.5min', '2', 'g'),
+            ('dev.unit-0.cachePeerRtt.three-net.5min', '3', 'g'),
         ])
 
     def test_peers_without_names(self):
@@ -185,7 +194,7 @@
         metrics = list(squid_metrics.get_metrics(self.config))
 
         self.assertEqual(metrics, [
-            ('dev.unit-0.cachePeerRtt.1.5min', '1', 1),
-            ('dev.unit-0.cachePeerRtt.2.5min', '2', 1),
-            ('dev.unit-0.cachePeerRtt.3.5min', '3', 1),
+            ('dev.unit-0.cachePeerRtt.1.5min', '1', 'g'),
+            ('dev.unit-0.cachePeerRtt.2.5min', '2', 'g'),
+            ('dev.unit-0.cachePeerRtt.3.5min', '3', 'g'),
         ])

=== modified file 'templates/metrics_cronjob.template'
--- templates/metrics_cronjob.template	2014-03-28 17:41:48 +0000
+++ templates/metrics_cronjob.template	2014-05-02 14:57:34 +0000
@@ -1,3 +1,3 @@
 # crontab for pushing squid metrics to statsd
-*/{{ interval }} * * * * root python {{ script }} {{ config }} | nc {{ statsd_host }} {{statsd_port }}
+*/{{ interval }} * * * * root python {{ script }} send
 

=== modified file 'templates/squid_metrics_ini.template'
--- templates/squid_metrics_ini.template	2014-02-25 17:21:46 +0000
+++ templates/squid_metrics_ini.template	2014-05-02 14:57:34 +0000
@@ -1,6 +1,7 @@
 [metrics]
 unit: {{ unit_name }}
 scheme: {{ metrics_scheme }}
-port: {{ snmp_port }}
+snmp_port: {{ snmp_port }}
+statsd_hostport: {{ metrics_target }}
 community: {{ snmp_community }}
 metrics: {{ metrics_list|join(',') }}