Merge lp:~adeuring/charmworld/more-heartbeat-info into lp:~juju-jitsu/charmworld/trunk

[Abel Deuring]

heartbeat: Checks added that the bundles collection is not empty, that all MongoDB collections exist; show size of ingest queues.

This branch adds several details to charmworld's heartbeat page:

- Number of bundles
- Do all required collections exist?
- Number of entries in the ingest queues

Implementation and test are quite straightforward, I think

[Curtis Hovey (sinzui)]

Thank you Abel. This is a good start. I think this branch is good to land.

Can you do a follow up branch to check some other collections?

REQUIRED BY PRODUCTION JUJU-GUI
featured
jenkins
qa

REQUIRED BY CHARMERS
review_queue
review_queue_latency
answer_queue
core_review_queue

KNOWING THE DB VERSION would be helpful
migration_version

review: Approve (code)

[Juju Gui Bot (juju-gui-bot)]

review: Approve (continuous-integration)

=== modified file 'charmworld/health.py'
--- charmworld/health.py	2013-08-16 17:55:44 +0000
+++ charmworld/health.py	2013-10-30 14:03:50 +0000
@@ -9,21 +9,33 @@
 FAIL = 'Fail'
 
 
-def check_mongodb(request):
+def _check_collection_size(request, collection_name):
+    """Return a Check that a mongodb collection is not empty."""
+    status = FAIL
+    try:
+        count = request.db[collection_name].count()
+        if count > 0:
+            remark = '{0} {1} found'.format(count, collection_name)
+            status = PASS
+        else:
+            remark = 'There are no {0}. Is ingest running?'
+            remark = remark.format(collection_name)
+    except:
+        remark = 'The mongodb {0} collection is not available.'
+        remark = remark.format(collection_name)
+    finally:
+        check = Check('{0} collection'.format(collection_name), status, remark)
+    return check
+
+
+def check_charms_collection(request):
     """Return a Check of the mongodb charms collection."""
-    status = FAIL
-    try:
-        count = request.db.charms.count()
-        if count > 0:
-            remark = '{0} charms found'.format(count)
-            status = PASS
-        else:
-            remark = 'There are no charms. Is ingest running?'
-    except:
-        remark = 'The mongodb charm collection is not available.'
-    finally:
-        check = Check('charms collection', status, remark)
-    return check
+    return _check_collection_size(request, 'charms')
+
+
+def check_bundles_collection(request):
+    """Return a Check of the mongodb bundles collection."""
+    return _check_collection_size(request, 'bundles')
 
 
 def check_elasticsearch(request):
@@ -83,3 +95,41 @@
     finally:
         check = Check('API3 interesting', status, remark)
     return check
+
+
+def check_collections(request):
+    """Return a Check of the collections that should exist in the Mongo DB."""
+    status = FAIL
+    required = set((
+        'basket-queue', 'baskets', 'bundles', 'charm-queue', 'charms'))
+    try:
+        existing = set(request.db.collection_names())
+        missing = required.difference(existing)
+        if missing:
+            missing = ', '.join(sorted(missing))
+            remark = 'Required collections do not (yet) exist: {0}'
+            remark = remark.format(missing)
+        else:
+            status = PASS
+            remark = 'All required collections exist.'
+    except:
+        remark = "Cannot query the MongoDB's collection names."
+    finally:
+        check = Check('MongoDB collections', status, remark)
+    return check
+
+
+def check_ingest_queues(request):
+    """Check the size of the ingest queues."""
+    status = FAIL
+    try:
+        charm_queue_size = request.db['charm-queue'].count()
+        basket_queue_size = request.db['basket-queue'].count()
+        status = PASS
+        remark = 'queued charms: {0}, queued baskets: {1}.'
+        remark = remark.format(charm_queue_size, basket_queue_size)
+    except:
+        remark = 'Cannot query the ingest queues.'
+    finally:
+        check = Check('Ingest queue sizes', status, remark)
+    return check
 
=== modified file 'charmworld/tests/test_health.py'
--- charmworld/tests/test_health.py	2013-08-15 21:31:37 +0000
+++ charmworld/tests/test_health.py	2013-10-30 14:03:50 +0000
@@ -6,8 +6,11 @@
 from charmworld.health import (
     check_api2,
     check_api3,
+    check_bundles_collection,
+    check_charms_collection,
+    check_collections,
     check_elasticsearch,
-    check_mongodb,
+    check_ingest_queues,
 )
 from charmworld.models import FeaturedSource
 from charmworld.testing import (
@@ -33,23 +36,40 @@
 
 class HealthTestCase(ViewTestBase, TestCheckMixin):
 
-    def test_check_mongodb_pass(self):
+    def test_check_charms_collection_pass(self):
         factory.makeCharm(self.db)
-        check = check_mongodb(self.getRequest())
+        check = check_charms_collection(self.getRequest())
         remark = '1 charms found'
         self.assertCheck(check, 'charms collection', 'Pass', remark)
 
-    def test_check_mongodb_missing_data_fail(self):
-        check = check_mongodb(self.getRequest())
+    def test_check_charms_collection_missing_data_fail(self):
+        check = check_charms_collection(self.getRequest())
         remark = 'There are no charms. Is ingest running?'
         self.assertCheck(check, 'charms collection', 'Fail', remark)
 
-    def test_check_mongodb_no_collection_fail(self):
+    def test_check_charms_collection_no_collection_fail(self):
         request = self.getRequest()
         with patch.object(request, 'db', new_callable=self.makeFailCallable):
-            check = check_mongodb(request)
+            check = check_charms_collection(request)
         self.assertCheck(check, 'charms collection', 'Fail')
 
+    def test_check_bundles_collection_pass(self):
+        factory.makeBundle(self.db)
+        check = check_bundles_collection(self.getRequest())
+        remark = '1 bundles found'
+        self.assertCheck(check, 'bundles collection', 'Pass', remark)
+
+    def test_check_bundles_collection_missing_data_fail(self):
+        check = check_bundles_collection(self.getRequest())
+        remark = 'There are no bundles. Is ingest running?'
+        self.assertCheck(check, 'bundles collection', 'Fail', remark)
+
+    def test_check_bundles_collection_no_collection_fail(self):
+        request = self.getRequest()
+        with patch.object(request, 'db', new_callable=self.makeFailCallable):
+            check = check_bundles_collection(request)
+        self.assertCheck(check, 'bundles collection', 'Fail')
+
     def test_check_elasticsearch_pass(self):
         charm_data = factory.makeCharm(self.db)[1]
         self.use_index_client()
@@ -112,3 +132,52 @@
 
     def test_check_api3_no_matches_fail(self):
         self._check_api_no_matches_fail(check_api3, 3)
+
+    def test_check_collections_pass(self):
+        request = self.getRequest()
+
+        def make_all_collection_names():
+            def all_collection_names():
+                return [
+                    'fs.chunks', 'system.indexes', 'basket-queue', 'baskets',
+                    'bundles', 'charm-queue', 'charms']
+            return all_collection_names
+
+        with patch.object(request.db, 'collection_names',
+                          new_callable=make_all_collection_names):
+            check = check_collections(request)
+        self.assertCheck(
+            check, 'MongoDB collections', 'Pass',
+            'All required collections exist.')
+
+    def test_check_collections_missing_collections(self):
+        request = self.getRequest()
+        check = check_collections(request)
+        self.assertCheck(
+            check, 'MongoDB collections', 'Fail',
+            'Required collections do not (yet) exist: basket-queue, baskets, '
+            'bundles, charm-queue, charms')
+
+    def test_check_collections_db_error(self):
+        request = self.getRequest()
+        with patch.object(request.db, 'collection_names',
+                          new_callable=self.makeFailCallable):
+            check = check_collections(request)
+        self.assertCheck(
+            check, 'MongoDB collections', 'Fail',
+            "Cannot query the MongoDB's collection names.")
+
+    def test_check_ingest_queues(self):
+        request = self.getRequest()
+        check = check_ingest_queues(request)
+        self.assertCheck(
+            check, 'Ingest queue sizes', 'Pass',
+            'queued charms: 0, queued baskets: 0.')
+
+    def test_check_ingest_queues_db_error(self):
+        request = self.getRequest()
+        with patch.object(request, 'db', new_callable=self.makeFailCallable):
+            check = check_ingest_queues(request)
+        self.assertCheck(
+            check, 'Ingest queue sizes', 'Fail',
+            'Cannot query the ingest queues.')
 
=== modified file 'charmworld/views/misc.py'
--- charmworld/views/misc.py	2013-08-16 00:29:11 +0000
+++ charmworld/views/misc.py	2013-10-30 14:03:50 +0000
@@ -9,8 +9,11 @@
 from charmworld.health import (
     check_api2,
     check_api3,
+    check_bundles_collection,
+    check_charms_collection,
+    check_collections,
     check_elasticsearch,
-    check_mongodb,
+    check_ingest_queues,
 )
 
 
@@ -48,8 +51,11 @@
 def heartbeat(request):
     """Return a dict summarising the operational state of charmworld."""
     checks = []
-    checks.append(check_mongodb(request))
+    checks.append(check_charms_collection(request))
+    checks.append(check_bundles_collection(request))
     checks.append(check_elasticsearch(request))
     checks.append(check_api2(request))
     checks.append(check_api3(request))
+    checks.append(check_collections(request))
+    checks.append(check_ingest_queues(request))
     return {'checks': checks}
 
=== modified file 'charmworld/views/tests/test_misc.py'
--- charmworld/views/tests/test_misc.py	2013-08-15 20:28:10 +0000
+++ charmworld/views/tests/test_misc.py	2013-10-30 14:03:50 +0000
@@ -109,18 +109,38 @@
     def test_all_checks_pass(self):
         # All checks pass when the services are available and data is returned.
         charm_data = factory.makeCharm(self.db, downloads=5)[1]
+        factory.makeBundle(self.db)
         self.use_index_client()
         self.index_client.index_charm(charm_data)
         FeaturedSource.from_db(self.db).set_featured(charm_data, 'charm')
-        response = heartbeat(self.getRequest())
+
+        def make_all_collection_names():
+            def all_collection_names():
+                return [
+                    'fs.chunks', 'system.indexes', 'basket-queue', 'baskets',
+                    'bundles', 'charm-queue', 'charms']
+            return all_collection_names
+
+        request = self.getRequest()
+        with patch.object(request.db, 'collection_names',
+                          new_callable=make_all_collection_names):
+            response = heartbeat(request)
+
         checks = response['checks']
+        self.assertEqual(7, len(checks))
         remark = '1 charms found'
         self.assertCheck(checks[0], 'charms collection', 'Pass', remark)
+        remark = '1 bundles found'
+        self.assertCheck(checks[1], 'bundles collection', 'Pass', remark)
         remark = '1 matches found'
-        self.assertCheck(checks[1], 'charms index', 'Pass', remark)
+        self.assertCheck(checks[2], 'charms index', 'Pass', remark)
         remark = 'Interesting has new, popular, and featured.'
-        self.assertCheck(checks[2], 'API2 interesting', 'Pass', remark)
-        self.assertCheck(checks[3], 'API3 interesting', 'Pass', remark)
+        self.assertCheck(checks[3], 'API2 interesting', 'Pass', remark)
+        self.assertCheck(checks[4], 'API3 interesting', 'Pass', remark)
+        remark = 'All required collections exist.'
+        self.assertCheck(checks[5], 'MongoDB collections', 'Pass', remark)
+        remark = 'queued charms: 0, queued baskets: 0.'
+        self.assertCheck(checks[6], 'Ingest queue sizes', 'Pass', remark)
 
     def test_checks_fail(self):
         # When services or data are not available, the checks fail.
@@ -128,11 +148,20 @@
         # are not available
         response = heartbeat(self.getRequest())
         checks = response['checks']
+        self.assertEqual(7, len(checks))
         remark = 'There are no charms. Is ingest running?'
         self.assertCheck(checks[0], 'charms collection', 'Fail', remark)
+        remark = 'There are no bundles. Is ingest running?'
+        self.assertCheck(checks[1], 'bundles collection', 'Fail', remark)
         remark = 'The elasticsearch charm index is not available.'
-        self.assertCheck(checks[1], 'charms index', 'Fail', remark)
+        self.assertCheck(checks[2], 'charms index', 'Fail', remark)
         remark = 'The API2 is not available.'
-        self.assertCheck(checks[2], 'API2 interesting', 'Fail', remark)
+        self.assertCheck(checks[3], 'API2 interesting', 'Fail', remark)
         remark = 'The API3 is not available.'
-        self.assertCheck(checks[3], 'API3 interesting', 'Fail', remark)
+        self.assertCheck(checks[4], 'API3 interesting', 'Fail', remark)
+        remark = (
+            'Required collections do not (yet) exist: basket-queue, '
+            'baskets, bundles, charm-queue, charms')
+        self.assertCheck(checks[5], 'MongoDB collections', 'Fail', remark)
+        remark = 'queued charms: 0, queued baskets: 0.'
+        self.assertCheck(checks[6], 'Ingest queue sizes', 'Pass', remark)