Merge lp:~adeuring/charmworld/more-heartbeat-info-2 into lp:charmworld

[Abel Deuring]

Add two more checks to the heartbeat page: Status of the ElasticSearch server ans the current bzr revision number.

This branch adds two more checks to the heartbeat page: Status of the ElasticSearch server ans the current bzr revision number.

When no charms are marked as new, popular of featured, the heartbeat page now shows some numbers.

There health module already had a function called check_elasticsearch() that tests if the charms index knows about at least one charm. While this is also in implicit check of the ES server, it is IMHO more a check if the _index_ is usable, so I renamed the function.

[Curtis Hovey (sinzui)]

Thank you Abel

review: Approve (code)

[Juju Gui Bot (juju-gui-bot)]

review: Approve (continuous-integration)

=== modified file 'charmworld/health.py'
--- charmworld/health.py	2013-10-30 13:20:01 +0000
+++ charmworld/health.py	2013-10-31 16:32:26 +0000
@@ -1,7 +1,10 @@
 # Copyright 2013 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
+from bzrlib.branch import Branch
+from bzrlib.transport import get_transport
 from collections import namedtuple
+from os.path import dirname
 
 
 Check = namedtuple('check', 'name status remark')
@@ -38,7 +41,7 @@
     return _check_collection_size(request, 'bundles')
 
 
-def check_elasticsearch(request):
+def check_charm_index(request):
     """Return a Check of the elasticsearch charms index."""
     status = FAIL
     try:
@@ -68,7 +71,12 @@
             status = PASS
             remark = 'Interesting has new, popular, and featured.'
         else:
-            remark = 'Interesting is missing new, popular, or featured.'
+            remark = (
+                'Interesting is missing new (found {0}), popular '
+                '(found {1}), or featured (found {2}).')
+            remark = remark.format(
+                len(interesting['new']), len(interesting['popular']),
+                len(interesting['featured']))
     except:
         remark = 'The API2 is not available.'
     finally:
@@ -89,7 +97,12 @@
             status = PASS
             remark = 'Interesting has new, popular, and featured.'
         else:
-            remark = 'Interesting is missing new, popular, or featured.'
+            remark = (
+                'Interesting is missing new (found {0}), popular '
+                '(found {1}), or featured (found {2}).')
+            remark = remark.format(
+                len(interesting['new']), len(interesting['popular']),
+                len(interesting['featured']))
     except:
         remark = 'The API3 is not available.'
     finally:
@@ -133,3 +146,42 @@
     finally:
         check = Check('Ingest queue sizes', status, remark)
     return check
+
+
+def check_bzr_revision(request):
+    """Return the current bzr revision."""
+    status = FAIL
+    try:
+        branch_dir = dirname(dirname(__file__))
+        transport = get_transport(branch_dir)
+        branch = Branch.open_from_transport(transport)
+        remark = str(branch.revno())
+        status = PASS
+    except:
+        remark = "Can't retrieve revision number."
+    finally:
+        check = Check('BZR revision', status, remark)
+    return check
+
+
+def check_elasticsearch_status(request):
+    """Check the health status of the ElasticSearch server."""
+    status = FAIL
+    try:
+        health = request.index_client.health()
+        remark = (
+            'status: {status}, '
+            'nodes: {number_of_nodes}, '
+            'active primary shards: {active_primary_shards}, '
+            'active shards: {active_shards}, '
+            'initializing shards: {initializing_shards}, '
+            'relocating shards: {relocating_shards}'
+        )
+        remark = remark.format(**health)
+        if health['status'] in ('green', 'yellow'):
+            status = PASS
+    except Exception:
+        remark = "Can't retrieve the ES server's health status."
+    finally:
+        check = Check('ElasticSearch server', status, remark)
+    return check

=== modified file 'charmworld/search.py'
--- charmworld/search.py	2013-09-16 21:14:06 +0000
+++ charmworld/search.py	2013-10-31 16:32:26 +0000
@@ -729,6 +729,10 @@
             copy.index_bundles(bundles)
             return copy
 
+    def health(self):
+        """The health status of the ES server."""
+        return self._client.health(self.index_name)
+
 
 def reindex(index_client, charms=None):
     """Reindex documents with the current mapping.

=== modified file 'charmworld/tests/test_health.py'
--- charmworld/tests/test_health.py	2013-10-30 13:20:01 +0000
+++ charmworld/tests/test_health.py	2013-10-31 16:32:26 +0000
@@ -7,9 +7,11 @@
     check_api2,
     check_api3,
     check_bundles_collection,
+    check_bzr_revision,
     check_charms_collection,
     check_collections,
-    check_elasticsearch,
+    check_charm_index,
+    check_elasticsearch_status,
     check_ingest_queues,
 )
 from charmworld.models import FeaturedSource
@@ -70,25 +72,25 @@
             check = check_bundles_collection(request)
         self.assertCheck(check, 'bundles collection', 'Fail')
 
-    def test_check_elasticsearch_pass(self):
+    def test_check_charm_index_pass(self):
         charm_data = factory.makeCharm(self.db)[1]
         self.use_index_client()
         self.index_client.index_charm(charm_data)
-        check = check_elasticsearch(self.getRequest())
+        check = check_charm_index(self.getRequest())
         remark = '1 matches found'
         self.assertCheck(check, 'charms index', 'Pass', remark)
 
-    def test_check_elasticsearch_missing_data_fail(self):
+    def test_check_charm_index_missing_data_fail(self):
         self.use_index_client()
         request = self.getRequest()
-        check = check_elasticsearch(request)
+        check = check_charm_index(request)
         remark = 'There are no matches. Is ingest running?'
         self.assertCheck(check, 'charms index', 'Fail', remark)
 
-    def test_check_elasticsearch_no_index_fail(self):
+    def test_check_charm_index_no_index_fail(self):
         request = self.getRequest()
         # The index is not setup.
-        check = check_elasticsearch(request)
+        check = check_charm_index(request)
         self.assertCheck(check, 'charms index', 'Fail')
 
     def _check_api_pass(self, function, version):
@@ -106,7 +108,9 @@
         self.index_client.index_charm(charm_data)
         # There are no featured or downloaded charms.
         check = function(self.getRequest())
-        remark = 'Interesting is missing new, popular, or featured.'
+        remark = (
+            'Interesting is missing new (found 1), popular (found 1), or '
+            'featured (found 0).')
         self.assertCheck(check, 'API%s interesting' % version, 'Fail', remark)
 
     def _check_api_no_matches_fail(self, function, version):
@@ -181,3 +185,79 @@
         self.assertCheck(
             check, 'Ingest queue sizes', 'Fail',
             'Cannot query the ingest queues.')
+
+    def test_check_bzr_revision(self):
+        request = self.getRequest()
+        check = check_bzr_revision(request)
+        self.assertCheck(check, 'BZR revision', 'Pass')
+        # check.remark is an integer: No exception is raised.
+        int(check.remark)
+
+    def test_check_bzr_revision_error(self):
+        request = self.getRequest()
+        from charmworld import health
+        with patch.object(health, 'get_transport',
+                          new_callable=self.makeFailCallable):
+            check = check_bzr_revision(request)
+        remark = "Can't retrieve revision number."
+        self.assertCheck(check, 'BZR revision', 'Fail', remark)
+
+    def test_check_elasticsearch_status(self):
+        self.use_index_client()
+        request = self.getRequest()
+        check = check_elasticsearch_status(request)
+        self.assertCheck(check, 'ElasticSearch server', 'Pass')
+        self.assertTrue(check.remark.startswith('status: green, nodes:'))
+
+    def test_check_elasticsearch_status_es_yellow_status(self):
+        index_client = self.use_index_client()
+
+        def make_health_method():
+            def health():
+                return {
+                    'status': 'yellow',
+                    'number_of_nodes': 1,
+                    'active_primary_shards': 0,
+                    'active_shards': 1,
+                    'initializing_shards': 0,
+                    'relocating_shards': 0,
+                }
+            return health
+
+        with patch.object(index_client, 'health',
+                          new_callable=make_health_method):
+            request = self.getRequest()
+            check = check_elasticsearch_status(request)
+        self.assertCheck(check, 'ElasticSearch server', 'Pass')
+        self.assertTrue(check.remark.startswith('status: yellow, nodes:'))
+
+    def test_check_elasticsearch_status_es_red_status(self):
+        index_client = self.use_index_client()
+
+        def make_health_method():
+            def health():
+                return {
+                    'status': 'red',
+                    'number_of_nodes': 1,
+                    'active_primary_shards': 0,
+                    'active_shards': 1,
+                    'initializing_shards': 0,
+                    'relocating_shards': 0,
+                }
+            return health
+
+        with patch.object(index_client, 'health',
+                          new_callable=make_health_method):
+            request = self.getRequest()
+            check = check_elasticsearch_status(request)
+        self.assertCheck(check, 'ElasticSearch server', 'Fail')
+        self.assertTrue(check.remark.startswith('status: red, nodes:'))
+
+    def test_check_elasticsearch_status_es_access_failing(self):
+        index_client = self.use_index_client()
+        with patch.object(index_client, 'health',
+                          new_callable=self.makeFailCallable):
+            request = self.getRequest()
+            check = check_elasticsearch_status(request)
+            remark = "Can't retrieve the ES server's health status."
+        self.assertCheck(check, 'ElasticSearch server', 'Fail', remark)

=== modified file 'charmworld/tests/test_search.py'
--- charmworld/tests/test_search.py	2013-10-24 17:24:43 +0000
+++ charmworld/tests/test_search.py	2013-10-31 16:32:26 +0000
@@ -904,6 +904,17 @@
         after = self.index_client.get_mapping()
         self.assertEqual(before, after)
 
+    def test_health(self):
+        # ElasticSearchClient.health() returns the health status of
+        # the index.
+        status = self.index_client.health()
+        expected_keys = set((
+            'active_primary_shards', 'active_shards', 'cluster_name',
+            'initializing_shards', 'number_of_data_nodes',
+            'number_of_nodes', 'relocating_shards', 'status', 'timed_out',
+            'unassigned_shards'))
+        self.assertEqual(expected_keys, set(status))
+
 
 class TestIndexingBundles(TestCase):
 

=== modified file 'charmworld/views/misc.py'
--- charmworld/views/misc.py	2013-10-30 13:20:01 +0000
+++ charmworld/views/misc.py	2013-10-31 16:32:26 +0000
@@ -10,9 +10,11 @@
     check_api2,
     check_api3,
     check_bundles_collection,
+    check_bzr_revision,
+    check_charm_index,
     check_charms_collection,
     check_collections,
-    check_elasticsearch,
+    check_elasticsearch_status,
     check_ingest_queues,
 )
 
@@ -53,9 +55,11 @@
     checks = []
     checks.append(check_charms_collection(request))
     checks.append(check_bundles_collection(request))
-    checks.append(check_elasticsearch(request))
+    checks.append(check_charm_index(request))
     checks.append(check_api2(request))
     checks.append(check_api3(request))
     checks.append(check_collections(request))
     checks.append(check_ingest_queues(request))
+    checks.append(check_bzr_revision(request))
+    checks.append(check_elasticsearch_status(request))
     return {'checks': checks}

=== modified file 'charmworld/views/tests/test_misc.py'
--- charmworld/views/tests/test_misc.py	2013-10-30 13:20:01 +0000
+++ charmworld/views/tests/test_misc.py	2013-10-31 16:32:26 +0000
@@ -127,7 +127,7 @@
             response = heartbeat(request)
 
         checks = response['checks']
-        self.assertEqual(7, len(checks))
+        self.assertEqual(9, len(checks))
         remark = '1 charms found'
         self.assertCheck(checks[0], 'charms collection', 'Pass', remark)
         remark = '1 bundles found'
@@ -141,14 +141,19 @@
         self.assertCheck(checks[5], 'MongoDB collections', 'Pass', remark)
         remark = 'queued charms: 0, queued baskets: 0.'
         self.assertCheck(checks[6], 'Ingest queue sizes', 'Pass', remark)
+        self.assertCheck(checks[7], 'BZR revision', 'Pass')
+        self.assertCheck(checks[8], 'ElasticSearch server', 'Pass')
 
     def test_checks_fail(self):
         # When services or data are not available, the checks fail.
         # No setup is performed, creating a case where services and data
         # are not available
-        response = heartbeat(self.getRequest())
+        from charmworld import health
+        with patch.object(health, 'get_transport',
+                          new_callable=self.makeFailCallable):
+            response = heartbeat(self.getRequest())
         checks = response['checks']
-        self.assertEqual(7, len(checks))
+        self.assertEqual(9, len(checks))
         remark = 'There are no charms. Is ingest running?'
         self.assertCheck(checks[0], 'charms collection', 'Fail', remark)
         remark = 'There are no bundles. Is ingest running?'
@@ -165,3 +170,7 @@
         self.assertCheck(checks[5], 'MongoDB collections', 'Fail', remark)
         remark = 'queued charms: 0, queued baskets: 0.'
         self.assertCheck(checks[6], 'Ingest queue sizes', 'Pass', remark)
+        remark = "Can't retrieve revision number."
+        self.assertCheck(checks[7], 'BZR revision', 'Fail', remark)
+        remark = "Can't retrieve the ES server's health status."
+        self.assertCheck(checks[8], 'ElasticSearch server', 'Fail', remark)