Merge ~hloeung/content-cache-charm:nagios-monitoring into content-cache-charm:master

[Haw Loeung]

Add configuring up NRPE and shipping out nagios checks

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Joel Sing (jsing)]

LGTM, some comments inline (mostly future follow up).

review: Approve (+1)

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision e33b2b4cb03fbe6cb7629e90086d58fd12f15545

diff --git a/README.md b/README.md
index 8ed7922..ad67998 100644
--- a/README.md
+++ b/README.md
@@ -9,3 +9,5 @@ Deploy your own content distribution network (CDN).
 # TODO
 - add nagios check / monitoring for configurable percentage of backends down
   - e.g. 80% critical, 50% warning
+- add removal of NRPE checks for sites that no longer exists.
+- add unconfigure_nagios() for when NRPE relation is destroyed/removed.
diff --git a/reactive/content_cache.py b/reactive/content_cache.py
index df7199f..fc6cffb 100644
--- a/reactive/content_cache.py
+++ b/reactive/content_cache.py
@@ -4,6 +4,7 @@ import yaml
 from charms import reactive
 from charms.layer import status
 from charmhelpers.core import hookenv, host
+from charmhelpers.contrib.charmsupport import nrpe
 
 from lib import nginx
 from lib import haproxy as HAProxy
@@ -36,6 +37,7 @@ def install():
 def config_changed():
     reactive.clear_flag('content_cache.haproxy.configured')
     reactive.clear_flag('content_cache.nginx.configured')
+    reactive.clear_flag('nagios-nrpe.configured')
 
 
 @reactive.when('content_cache.nginx.configured', 'content_cache.haproxy.configured')
@@ -135,6 +137,55 @@ def configure_haproxy():
     reactive.set_flag('content_cache.haproxy.configured')
 
 
[e-mail]('content_cache.nginx.configured', 'content_cache.haproxy.configured')
[e-mail]('nrpe-external-master.available')
[e-mail]_not('nagios-nrpe.configured')
+def configure_nagios():
+    status.maintenance('setting up NRPE checks')
+
+    config = hookenv.config()
+
+    # Use charmhelpers.contrib.charmsupport's nrpe to determine hostname
+    hostname = nrpe.get_nagios_hostname()
+    nrpe_setup = nrpe.NRPE(hostname=hostname, primary=True)
+
+    conf = yaml.safe_load(config.get('sites'))
+    cache_port = 0
+    backend_port = 0
+    for site in conf.keys():
+        (cache_port, backend_port) = next_port_pair(cache_port, backend_port)
+
+        default_port = 80
+        url = 'http://{}'.format(site)
+        tls_cert_bundle_path = conf[site].get('tls-cert-bundle-path')
+        tls = ''
+        if tls_cert_bundle_path:
+            default_port = 443
+            url = 'https://{}'.format(site)
+            tls = ' -S --sni'
+
+        # Listen / frontend check
+        check_name = 'site_{}_listen'.format(generate_nagios_check_name(site))
+        cmd = '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H {site} -p {port}{tls} -u {url} -j GET' \
+              .format(site=site, port=default_port, url=url, tls=tls)
+        nrpe_setup.add_check(check_name, '{} site listen check'.format(site), cmd)
+
+        # Cache layer check
+        check_name = 'site_{}_cache'.format(generate_nagios_check_name(site))
+        cmd = '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H {site} -p {cache_port} -u {url} -j GET' \
+              .format(site=site, cache_port=cache_port, url=url)
+        nrpe_setup.add_check(check_name, '{} cache check'.format(site), cmd)
+
+        # Backend proxy layer check
+        check_name = 'site_{}_backend_proxy'.format(generate_nagios_check_name(site))
+        cmd = '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H {site} -p {backend_port} -u {url} -j GET' \
+              .format(site=site, backend_port=backend_port, url=url)
+        nrpe_setup.add_check(check_name, '{} backend proxy check'.format(site), cmd)
+
+    nrpe_setup.write()
+    reactive.set_flag('nagios-nrpe.configured')
+
+
 class InvalidPortError(Exception):
     pass
 
@@ -164,3 +215,7 @@ def next_port_pair(cache_port, backend_port,
         raise InvalidPortError('Dynamically allocated backend_port out of range')
 
     return (cache_port, backend_port)
+
+
+def generate_nagios_check_name(site):
+    return site.replace('.', '_').replace('-', '_')
diff --git a/tests/unit/test_content_cache.py b/tests/unit/test_content_cache.py
index 13b668e..1623fe1 100644
--- a/tests/unit/test_content_cache.py
+++ b/tests/unit/test_content_cache.py
@@ -41,7 +41,7 @@ class TestCharm(unittest.TestCase):
         patcher = mock.patch('charmhelpers.core.hookenv.config')
         self.mock_config = patcher.start()
         self.addCleanup(patcher.stop)
-        self.mock_config.return_value = {}
+        self.mock_config.return_value = {'nagios_context': 'juju'}
 
         patcher = mock.patch('multiprocessing.cpu_count')
         self.mock_cpu_count = patcher.start()
@@ -168,6 +168,46 @@ class TestCharm(unittest.TestCase):
             content_cache.configure_haproxy()
             self.assertFalse(service_start_or_restart.assert_not_called())
 
+    @mock.patch('charmhelpers.contrib.charmsupport.nrpe.get_nagios_hostname')
+    @mock.patch('charmhelpers.contrib.charmsupport.nrpe.NRPE')
+    def test_configure_nagios(self, nrpe, get_nagios_hostname):
+        get_nagios_hostname.return_value = 'some-host.local'
+        with open('tests/unit/files/config_test_config.txt', 'r', encoding='utf-8') as f:
+            config = f.read()
+        self.mock_config.return_value = {'sites': config}
+        nrpe_instance_mock = nrpe(get_nagios_hostname(), primary=True)
+        content_cache.configure_nagios()
+        expected = [mock.call('site_site1_local_listen', 'site1.local site listen check',
+                              '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H site1.local -p 80'
+                              ' -u http://site1.local -j GET'),
+                    mock.call('site_site1_local_cache', 'site1.local cache check',
+                              '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H site1.local -p 6080'
+                              ' -u http://site1.local -j GET'),
+                    mock.call('site_site1_local_backend_proxy', 'site1.local backend proxy check',
+                              '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H site1.local -p 8080'
+                              ' -u http://site1.local -j GET')]
+        self.assertFalse(nrpe_instance_mock.add_check.assert_has_calls(expected, any_order=True))
+        expected = [mock.call('site_site2_local_listen', 'site2.local site listen check',
+                              '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H site2.local -p 443 -S --sni'
+                              ' -u https://site2.local -j GET'),
+                    mock.call('site_site2_local_cache', 'site2.local cache check',
+                              '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H site2.local -p 6081'
+                              ' -u https://site2.local -j GET'),
+                    mock.call('site_site2_local_backend_proxy', 'site2.local backend proxy check',
+                              '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H site2.local -p 8081'
+                              ' -u https://site2.local -j GET')]
+        self.assertFalse(nrpe_instance_mock.add_check.assert_has_calls(expected, any_order=True))
+        expected = [mock.call('site_site3_local_listen', 'site3.local site listen check',
+                              '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H site3.local -p 80'
+                              ' -u http://site3.local -j GET'),
+                    mock.call('site_site3_local_cache', 'site3.local cache check',
+                              '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H site3.local -p 6082'
+                              ' -u http://site3.local -j GET'),
+                    mock.call('site_site3_local_backend_proxy', 'site3.local backend proxy check',
+                              '/usr/lib/nagios/plugins/check_http -I 127.0.0.1 -H site3.local -p 8082'
+                              ' -u http://site3.local -j GET')]
+        self.assertFalse(nrpe_instance_mock.add_check.assert_has_calls(expected, any_order=True))
+
     def test_next_port_pair(self):
         self.assertEqual(content_cache.next_port_pair(0, 0),
                          (content_cache.BASE_CACHE_PORT, content_cache.BASE_BACKEND_PORT))
@@ -205,6 +245,9 @@ class TestCharm(unittest.TestCase):
             content_cache.next_port_pair(0, content_cache.BACKEND_PORT_LIMIT,
                                          backend_port_limit=content_cache.BASE_BACKEND_PORT+10)
 
+    def test_generate_nagios_check_name(self):
+        self.assertEqual(content_cache.generate_nagios_check_name('site-1.local'), 'site_1_local')
+
 
 if __name__ == '__main__':
     unittest.main()