Merge lp:~abentley/charmworld/store-data-on-enqueue into lp:~juju-jitsu/charmworld/trunk

[Aaron Bentley]

Get store data when querying lp.

This branch ensures we get the store data at the same time we get the LP data, so that we have all we need when we create the mongodb primary key

[lp:~abentley/charmworld/store-data-on-enqueue]

317. By Aaron Bentley on 2013-07-22

Remove requests import.

[Curtis Hovey (sinzui)]

Thank you. This can land once you reconcile the argument changes show in the conflicts.

review: Approve (code)

[lp:~abentley/charmworld/store-data-on-enqueue]

318. By Aaron Bentley on 2013-07-22

Merged trunk into store-data-on-enqueue.

=== modified file 'charmworld/charmstore.py'
--- charmworld/charmstore.py	2013-07-01 14:45:02 +0000
+++ charmworld/charmstore.py	2013-07-22 20:55:30 +0000
@@ -7,7 +7,6 @@
 from datetime import timedelta
 
 
-import requests
 from requests import (
     Request,
     Session,
@@ -71,12 +70,15 @@
             params=params
         )
 
-    def get_charm_info(self, address):
-        url = self.STORE_URL + "/charm-info?charms=%s&stats=0" % address
-        data = requests.get(url).json()
-        data = data[address]
-        data['address'] = address
-        return data
+    @classmethod
+    def get_charms_info_request(cls, addresses):
+        return Request('POST', cls.STORE_URL + '/charm-info',
+                       data={'stats': '0', 'charms': addresses})
+
+    def get_charms_info(self, addresses):
+        request = self.get_charms_info_request(addresses)
+        response = self.session.send(request.prepare())
+        return response.json()
 
     def get_download_counts(self, charm, start=None, end=None):
         """Retrieve download counts for each day in the specified range.
@@ -106,3 +108,24 @@
         start = end - timedelta(days=days)
         counts = self.get_download_counts(charm, start, end)
         return sum(downloads for date, downloads in counts)
+
+
+def get_store_charm_info(store, charms):
+    addresses = []
+    charm_addresses = []
+    for charm in charms:
+        possible_addresses = {'long': get_address(charm, False)}
+        if charm['promulgated']:
+            possible_addresses['short'] = get_address(charm, True)
+        charm_addresses.append(possible_addresses)
+        addresses.extend(possible_addresses.values())
+    info = store.get_charms_info(addresses)
+    result = []
+    for possible_addresses in charm_addresses:
+        if ('short' in possible_addresses and
+            'errors' not in info[possible_addresses['short']]):
+            address = possible_addresses['short']
+        else:
+            address = possible_addresses['long']
+        result.append((address, info[address]))
+    return result

=== modified file 'charmworld/jobs/ingest.py'
--- charmworld/jobs/ingest.py	2013-07-19 20:14:26 +0000
+++ charmworld/jobs/ingest.py	2013-07-22 20:55:30 +0000
@@ -225,15 +225,17 @@
     return True
 
 
-def update_charm(charm_data, db):
+def update_charm(charm_data, db, store):
     charm_data["_id"] = charm_data["branch_spec"]
     # Drop existing error data so that charms can lose their error status.
     charm_data.pop('error', None)
+    if 'errors' in charm_data.get('store_data', {}):
+        return
     log = logging.getLogger("charm.update_charm")
     fs = getfs(db)
     try:
-        update_from_store(charm_data, CharmStore(), log)
         do_bzr_update(charm_data, db, fs, log)
+        update_download_count(store, charm_data)
         update_proof_data(charm_data, log)
         update_jenkins_data(db, charm_data, log)
         update_from_revisions(charm_data)
@@ -257,7 +259,7 @@
         if charm_data is None:
             charm_data = {}
         charm_data.update(payload)
-        update_charm(charm_data, self.db)
+        update_charm(charm_data, self.db, CharmStore())
         index_client = ElasticSearchClient.from_settings(settings)
         self.log.info('Saving %s' % charm_data['branch_spec'])
         CharmSource(self.db, index_client).save(charm_data)
@@ -689,24 +691,13 @@
     charm['date_created'] = timestamp(date_created.replace(microsecond=0))
 
 
-def update_from_store(charm, store, log):
-    old_address = None
-    for address in addresses(charm):
-        if old_address is not None:
-            log.info("rechecking %s with %s", old_address, address)
-        old_address = address
-        data = store.get_charm_info(address)
-        if 'errors' not in data and 'warnings' not in data:
-            break
-
+def update_from_store(charm, address, data, check_time, log):
     if 'errors' in data or 'warnings' in data:
         log.warning("store error on %s %s" % (address, data))
-
-    data["store_checked"] = datetime.now().ctime()
-
+    data["store_checked"] = check_time
+    charm['address'] = address
     charm['store_data'] = data
     charm['store_url'] = make_store_url(data['revision'], address)
-    update_download_count(store, charm)
 
 
 # XXX j.c.sackett Jan 31 2013 Bug:1111708 scan_repo is swapped for

=== modified file 'charmworld/jobs/lp.py'
--- charmworld/jobs/lp.py	2013-07-22 17:57:14 +0000
+++ charmworld/jobs/lp.py	2013-07-22 20:55:30 +0000
@@ -3,9 +3,17 @@
 # the file LICENSE).
 
 import argparse
+from datetime import (
+    datetime,
+)
 import logging
 import pymongo
 
+from charmworld.charmstore import (
+    CharmStore,
+    get_store_charm_info,
+)
+from charmworld.jobs.ingest import update_from_store
 from charmworld.lp import (
     BASKET_SERIES,
     get_branch_tips,
@@ -52,7 +60,7 @@
                 baskets.append(data)
             continue
 
-        if data["bname"] not in ("trunk", "trunk-1"):
+        if data["bname"] != "trunk":
             log.debug("Skipped branch %s", repo)
             continue
 
@@ -103,12 +111,17 @@
             break
 
 
-def queue_from_branches(db, charm_queue, basket_queue, limit=None,
-                        import_filter=default):
+def queue_from_branches(db, store, charm_queue, basket_queue, limit=None,
+                        import_filter=default, check_time=None):
     log = logging.getLogger("charm.launchpad")
     charm_data, baskets = all_branch_data()
-    for charm in all_charms(db, charm_data, limit=limit,
-                            import_filter=import_filter):
+    charms = list(all_charms(db, charm_data, limit=limit,
+                             import_filter=import_filter))
+    charms_info = get_store_charm_info(store, charms)
+    if check_time is None:
+        check_time = datetime.now().ctime()
+    for charm, (address, store_data) in zip(charms, charms_info):
+        update_from_store(charm, address, store_data, check_time, log)
         added = charm_queue.put(charm)
         if added and 0:
             log.info("Queued %s", charm)
@@ -167,7 +180,7 @@
                   db, log):
             charm_queue = get_queue(db, CHARM_QUEUE)
             basket_queue = get_queue(db, BASKET_QUEUE)
-            queue_from_branches(db, charm_queue, basket_queue,
+            queue_from_branches(db, CharmStore(), charm_queue, basket_queue,
                                 charm_import_limit, args.prefix)
     except LockHeld, error:
         log.warn(str(error))

=== modified file 'charmworld/jobs/tests/test_ingest.py'
--- charmworld/jobs/tests/test_ingest.py	2013-07-19 15:05:00 +0000
+++ charmworld/jobs/tests/test_ingest.py	2013-07-22 20:55:30 +0000
@@ -22,7 +22,10 @@
 from mock import patch
 from pyelasticsearch.exceptions import ElasticHttpNotFoundError
 
-from charmworld.charmstore import get_address
+from charmworld.charmstore import (
+    CharmStore,
+    get_address,
+)
 from charmworld.jobs.config import CHARM_DIR
 from charmworld.jobs.ingest import (
     IngestError,
@@ -415,7 +418,7 @@
     def test_update_charm_does_not_index(self):
         charm_data = factory.get_charm_json()
         with charm_update_environment(charm_data, self.use_index_client()):
-            update_charm(charm_data, self.db)
+            update_charm(charm_data, self.db, CharmStore())
         self.index_client.wait_for_startup()
         with self.assertRaises(ElasticHttpNotFoundError):
             self.index_client.get(charm_data['_id'])
@@ -423,16 +426,33 @@
     def test_update_charm_does_not_update_mongo(self):
         charm_data = factory.get_charm_json()
         with charm_update_environment(charm_data, self.use_index_client()):
-            update_charm(charm_data, self.db)
+            update_charm(charm_data, self.db, CharmStore())
         self.assertIs(None, self.db.charms.find_one(charm_data['_id']))
 
     def test_forgets_past_error(self):
         charm_data = factory.get_charm_json(promulgated=True)
         charm_data['error'] = {'error_stage': 'foo', 'error': 'bar'}
         with charm_update_environment(charm_data, self.use_index_client()):
-            update_charm(charm_data, self.db)
+            update_charm(charm_data, self.db, CharmStore())
         self.assertNotIn('error', charm_data)
 
+    def test_updates_download_info(self):
+        charm_data = factory.get_charm_json()
+        del charm_data['downloads']
+
+        class FakeCharmStore:
+            @staticmethod
+            def count_downloads_in_days(charm, days, end):
+                return 0
+
+            @staticmethod
+            def get_download_counts(charm, start=None, end=None):
+                return [[5]]
+
+        with charm_update_environment(charm_data, self.use_index_client()):
+            update_charm(charm_data, self.db, FakeCharmStore)
+        self.assertEqual(5, charm_data['downloads'])
+
 
 class TestJob(IngestJob):
     name = 'test'

=== modified file 'charmworld/jobs/tests/test_lp.py'
--- charmworld/jobs/tests/test_lp.py	2013-07-22 17:57:14 +0000
+++ charmworld/jobs/tests/test_lp.py	2013-07-22 20:55:30 +0000
@@ -26,10 +26,13 @@
     queue_from_branches,
 )
 from charmworld.jobs.utils import get_queue
+from charmworld.tests.test_charmstore import FakeCharmStore
 
 
 def all_branch_data_mock(charm_data=None, limit=None, import_filter=None):
-    return [{'branch_dir': 'foo', 'branch_spec': 'bar'}], [{'foo': 'bar'}]
+    return ([{'branch_dir': 'foo', 'branch_spec': 'bar', 'owner': 'baz',
+             'series': 'qux', 'name': 'quxx', 'promulgated': False}],
+            [{'foo': 'bar'}])
 
 
 def failing_requests_get(*args, **kwargs):
@@ -127,8 +130,8 @@
                          [charm['branch_spec'] for charm in charms])
 
     def test_all_branch_data_not_trunk(self):
-        # If the branch is not trunk or trunk-1, the charm is not returned by
-        # available charms.
+        # If the branch is not trunk, the charm is not returned by
+        # all_branch_data.
         handler = self.get_handler("charm.launchpad")
         charm_data = [[u'~charmers/charms/precise/someproject/foo',
                        u'[e-mail]-20120329093714-s2m9e28dwotmijqc',
@@ -261,10 +264,26 @@
         charm_queue = get_queue(self.db, 'test_charm_queue')
         basket_queue = get_queue(self.db, 'test_basket_queue')
         self.addCleanup(basket_queue.clear)
-        queue_from_branches(self.db, charm_queue, basket_queue)
+        queue_from_branches(self.db, FakeCharmStore(), charm_queue,
+                            basket_queue, check_time='lambada')
         item = charm_queue.next()
+        expected = {
+            'address': 'cs:~baz/qux/quxx',
+            'branch_dir': 'foo',
+            'branch_spec': 'bar',
+            'name': 'quxx',
+            'owner': 'baz',
+            'promulgated': False,
+            'series': 'qux',
+            'store_data': {
+                'errors': ['entry not found'],
+                'revision': 0,
+                'store_checked': 'lambada'
+            },
+            'store_url': 'cs:~baz/qux/quxx-0',
+        }
         self.assertEqual(
-            {'branch_dir': 'foo', 'branch_spec': 'bar'}, item.payload)
+            expected, item.payload)
         item = basket_queue.next()
         self.assertEqual({'foo': 'bar'}, item.payload)
 
@@ -293,6 +312,6 @@
         out_queue = get_queue(self.db, 'test_queue')
         factory.makeCharm(self.db)
         self.assertRaises(
-            ConnectionError, queue_from_branches, self.db,
+            ConnectionError, queue_from_branches, self.db, FakeCharmStore(),
             out_queue, out_queue)
         self.assertEqual(0, out_queue.size())

=== modified file 'charmworld/jobs/tests/test_store.py'
--- charmworld/jobs/tests/test_store.py	2013-06-24 16:32:22 +0000
+++ charmworld/jobs/tests/test_store.py	2013-07-22 20:55:30 +0000
@@ -2,12 +2,12 @@
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 from logging import getLogger
-from mock import patch
 from requests import Response
 
 from charmworld.charmstore import CharmStore
 from charmworld.jobs.ingest import (
     addresses,
+    get_address,
     update_from_store,
 )
 from charmworld.testing import factory
@@ -45,37 +45,23 @@
         store = CharmStore()
         self.falsify_session(store)
         ignore, charm = factory.makeCharm(self.db, promulgated=True)
-        with patch.object(store, 'get_charm_info',
-                          lambda x: self._mock_data(error=True)):
-            update_from_store(charm, store, self.log)
-
-        first_address = "cs:%s/%s" % (charm["series"], charm["name"])
-        second_address = "cs:~%s/%s/%s" % (
-            charm["owner"], charm["series"], charm["name"])
+        address = get_address(charm, True)
+        update_from_store(charm, address, self._mock_data(error=True), 'time',
+                          self.log)
         log_messages = [record.getMessage() for record in handler.buffer]
-        err_msg = "rechecking %s with %s" % (first_address, second_address)
-        self.assertIn(err_msg, log_messages)
         err_msg = ("store error on %s {'errors': 'Error forced by mock.',"
-                   " 'revision': 1}" % second_address)
+                   " 'revision': 1}" % address)
         self.assertIn(err_msg, log_messages)
 
     def test_error_handling_warning(self):
-        handler = self.get_handler('charm.store')
-        store = CharmStore()
-        self.falsify_session(store)
+        handler = self.get_handler(self.log.name)
         ignore, charm = factory.makeCharm(self.db, promulgated=True)
-        with patch.object(store, 'get_charm_info',
-                          lambda x: self._mock_data(warning=True)):
-            update_from_store(charm, store, self.log)
-
-        first_address = "cs:%s/%s" % (charm["series"], charm["name"])
-        second_address = "cs:~%s/%s/%s" % (
-            charm["owner"], charm["series"], charm["name"])
+        address = get_address(charm, True)
+        update_from_store(charm, address, self._mock_data(warning=True),
+                          'time', self.log)
         log_messages = [record.getMessage() for record in handler.buffer]
-        err_msg = "rechecking %s with %s" % (first_address, second_address)
-        self.assertIn(err_msg, log_messages)
         err_msg = ("store error on %s {'warnings': 'Warning forced by mock.',"
-                   " 'revision': 1}" % second_address)
+                   " 'revision': 1}" % address)
         self.assertIn(err_msg, log_messages)
 
     def test_run(self):
@@ -83,16 +69,14 @@
         self.falsify_session(store, '[["2010-12-24", 4], ["2010-12-25", 1]]')
         ignore, charm = factory.makeCharm(self.db, promulgated=True)
         del charm['store_url']
-        with patch.object(store, 'get_charm_info',
-                          lambda x: self._mock_data()):
-            update_from_store(charm, store, self.log)
+        update_from_store(charm, get_address(charm, True), self._mock_data(),
+                          'foo', self.log)
         data = charm['store_data']
         self.assertIn('store_checked', data.keys())
         self.assertEqual(1, data['revision'])
         self.assertEqual(
             "cs:%s/%s-%d" % (charm["series"], charm["name"], 1),
             charm['store_url'])
-        self.assertEqual(5, charm['downloads_in_past_30_days'])
 
     def test_addresses(self):
         # The helper function addresses() returns the cs:~owner variant

=== modified file 'charmworld/models.py'
--- charmworld/models.py	2013-07-22 17:57:14 +0000
+++ charmworld/models.py	2013-07-22 20:55:30 +0000
@@ -334,7 +334,7 @@
     def bname(self):
         """The charm's branch name.
 
-        Only branches named trunk or trunk-1 are ingested from Launchpad.
+        Only branches named trunk are ingested from Launchpad.
         """
         return self._representation['bname']
 

=== modified file 'charmworld/tests/test_charmstore.py'
--- charmworld/tests/test_charmstore.py	2013-07-01 14:45:02 +0000
+++ charmworld/tests/test_charmstore.py	2013-07-22 20:55:30 +0000
@@ -8,17 +8,37 @@
     date,
     datetime,
 )
+import json
 import urlparse
 
 from requests import Response
 
-from charmworld.charmstore import CharmStore
+from charmworld.charmstore import (
+    CharmStore,
+    get_address,
+    get_store_charm_info,
+)
 from charmworld.testing import (
     factory,
     TestCase,
 )
 
 
+class FakeSessionBase:
+
+    @classmethod
+    def send(cls, request):
+        parts = urlparse.urlsplit(request.url)
+        query = urlparse.parse_qs(parts.query)
+        if request.body is not None:
+            data = urlparse.parse_qs(request.body)
+        else:
+            data = None
+        response = Response()
+        response._content = cls.parsed_send(request, parts, query, data)
+        return response
+
+
 class TestCharmStore(TestCase):
 
     def assertStoreURL(self, expected_path, url):
@@ -69,22 +89,18 @@
         end = date(2012, 12, 31)
         cs = CharmStore()
 
-        class FakeSession:
+        class FakeSession(FakeSessionBase):
 
             @staticmethod
-            def send(request):
-                parts = urlparse.urlsplit(request.url)
+            def parsed_send(request, parts, query, data):
                 self.assertEqual('/stats/counter/charm-bundle:foo:bar:baz',
                                  parts.path)
-                query = urlparse.parse_qs(parts.query)
                 self.assertEqual({
                     'start': ['2011-01-02'],
                     'end': ['2012-12-31'],
                     'by': ['day'],
                     'format': ['json']}, query)
-                response = Response()
-                response._content = '[[0]]'
-                return response
+                return '[[0]]'
 
         cs.session = FakeSession()
         result = cs.get_download_counts(charm, start, end)
@@ -123,3 +139,88 @@
         cs.session = FakeSession()
 
         self.assertEqual(5, cs.count_downloads_in_days(charm, 30, today))
+
+    def test_get_charms_info_request(self):
+        req = CharmStore.get_charms_info_request(
+            ['cs:asdf/qsf', 'cs:~foo/bar/baz'])
+        self.assertEqual('POST', req.method)
+        self.assertEqual(CharmStore.STORE_URL + '/charm-info', req.url)
+        self.assertEqual('0', req.data['stats'])
+        self.assertEqual(['cs:asdf/qsf', 'cs:~foo/bar/baz'],
+                         req.data['charms'])
+
+    def test_get_charms_info(self):
+        class FakeSession(FakeSessionBase):
+            @staticmethod
+            def parsed_send(req, parts, query, data):
+                self.assertEqual(
+                    data['charms'], ['cs:asdf/qsf', 'cs:~foo/bar/baz'])
+                return json.dumps({})
+        store = CharmStore()
+        store.session = FakeSession
+        info = store.get_charms_info(['cs:asdf/qsf', 'cs:~foo/bar/baz'])
+        self.assertEqual({}, info)
+
+
+MISSING_CHARM = {'revision': 0, 'errors': ['entry not found']}
+
+
+class FakeCharmStore:
+
+    def __init__(self, store_data=None):
+        if store_data is None:
+            store_data = {}
+        self.store_data = store_data
+
+    def get_charms_info(self, addresses):
+        return dict((a, self.store_data.get(a, MISSING_CHARM))
+                    for a in addresses)
+
+
+class TestGetStoreCharmInfo(TestCase):
+
+    def test_get_store_charm_info(self):
+        payload1 = factory.get_payload_json()
+        addr1 = get_address(payload1, False)
+        payload2 = factory.get_payload_json()
+        addr2 = get_address(payload2, False)
+        payload3 = factory.get_payload_json()
+        addr3 = get_address(payload3, False)
+        store = FakeCharmStore({addr1: {'1'}, addr2: {'2'}})
+        result = get_store_charm_info(store, [payload1, payload2, payload3])
+        self.assertEqual(
+            [(addr1, {'1'}), (addr2, {'2'}), (addr3, MISSING_CHARM)], result)
+        result = get_store_charm_info(store, [payload3, payload1, payload2])
+        self.assertEqual(
+            [(addr3, MISSING_CHARM), (addr1, {'1'}), (addr2, {'2'})], result)
+
+    def test_short_address(self):
+        payload = factory.get_payload_json(promulgated=True)
+        short_addr = get_address(payload, True)
+        store = FakeCharmStore({short_addr: {'foo': 'bar'}})
+        result = get_store_charm_info(store, [payload])
+        self.assertEqual([(short_addr, {'foo': 'bar'})], result)
+
+    def test_long_address_if_not_promulgated(self):
+        payload = factory.get_payload_json(promulgated=False)
+        short_addr = get_address(payload, True)
+        long_addr = get_address(payload, False)
+        store = FakeCharmStore(
+            {short_addr: {'foo': 'bar'}, long_addr: {'foo': 'bar'}})
+        result = get_store_charm_info(store, [payload])
+        self.assertEqual([(long_addr, {'foo': 'bar'})], result)
+
+    def test_long_address_if_short_not_in_store(self):
+        payload = factory.get_payload_json(promulgated=True)
+        long_addr = get_address(payload, False)
+        store = FakeCharmStore(
+            {long_addr: {'foo': 'bar'}})
+        result = get_store_charm_info(store, [payload])
+        self.assertEqual([(long_addr, {'foo': 'bar'})], result)
+
+    def test_long_address_if_neither_in_store(self):
+        payload = factory.get_payload_json(promulgated=True)
+        long_addr = get_address(payload, False)
+        store = FakeCharmStore({})
+        result = get_store_charm_info(store, [payload])
+        self.assertEqual([(long_addr, MISSING_CHARM)], result)

=== removed file 'migrations/versions/001_load_qa_questions.py'
--- migrations/versions/001_load_qa_questions.py	2013-06-04 15:36:45 +0000
+++ migrations/versions/001_load_qa_questions.py	1970-01-01 00:00:00 +0000
@@ -1,137 +0,0 @@
-# Copyright 2012, 2013 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# -*- coding: utf-8 -*-
-"""
-Load qa questions into the collection for populating the scoring parts of
-the quality assessment form.
-
-"""
-
-
-def upgrade(db, index_client):
-    """Complete this function with work to be done for the migration/update.
-
-    db is the pymongo db instance for our datastore. Charms are in db.charms
-    for instance.
-    """
-
-    initial = [
-        (
-            u'reliable',
-            u'Reliable',
-            [
-                (u'AWS', 1, u''),
-                (u'HP Cloud', 1, u''),
-                (u'OpenStack', 1, u''),
-                (u'LXC', 1, u''),
-                (u'MAAS', 1, u''),
-                (u'Check for integrity from upstream source', 1, u''),
-                (u'Fail gracefully if upstream source goes missing', 1, u''),
-                (u'Contain a suite of tests with the charm that pass', 1, u''),
-                (u'Passes tests from Jenkins on jujucharms.com', 1, u''),
-            ]
-        ),
-
-        (
-            u'secure',
-            u'Secure',
-            [
-                (u'Contain a well tested AppArmor profile', 1, ''),
-                (u'Conform to security policies of the charm store', 1, 'Tight access control'),
-                (u"Doesn't run as root", 1, ''),
-                (u'Per instance or service access control', 1, ''),
-            ]
-        ),
-
-        (
-            u'flexible',
-            u'Flexible',
-            [
-                (
-                    u'Contain opinionated tuning options', 1,
-                    u"""
-                    Examples (depends on the service): "safe", "default", "fast",
-                    "real fast, not so safe". Don't expose every configuration,
-                    "pick that reflect real world usage. Make it so I don't have to read the book.
-                    """
-                ),
-                (u'Use existing interfaces with other charms', 1, u'Highly relatable'),
-            ]
-        ),
-
-        (
-            u'data_handling',
-            u'Data Handling',
-            [
-                (u'Integrate data storage best practices', 1, u'Backups based on service usage'),
-                (u"Handle the service's user data", 1, u'Version control'),
-                (u"Handle the service's user data", 1, u'Automated snapshots and backup.'),
-            ]
-        ),
-
-        (
-            u'scalable',
-            u'Scaleable',
-            [
-                (u"Responds to add-unit based on the service's needs", 1,
-                    u'Configuration should not require additional steps to scale horizontally'),
-                (u'Be tested with a real workload, not just a synthetic benchmark', 1, ''),
-                (u'From upstream and existing devops practices for that service', 1, ''),
-                (u'Community peer reviewed', 1, ''),
-                (u'Have a configure option for most performant configuration if not the default', 1, ''),
-            ]
-        ),
-
-        (
-            u'easy_deploy',
-            u'Easy to Deploy',
-            [
-                (u'README with examples of use for a typical workload', 1, ''),
-                (u'README with examples of use for workloads at scale', 1, ''),
-                (u'README with examples of use recommend best-practice relationships', 1, ''),
-                (u'Allow installation from pure upstream source', 1, ''),
-                (u'Allow installation from your local source', 1, ''),
-                (u'Allow installation from PPA (if available)', 1, ''),
-                (u'Allow installation from the Ubuntu repository', 1, ''),
-            ]
-        ),
-
-        (
-            u'responsive',
-            u'Responsive to DevOps Needs',
-            [
-                (u'Allow for easy upgrade via juju upgrade-charm', 1, ''),
-                (u'Allow upgrading the service itself.', 1, ''),
-                (u'Responsive to user bug reports and concerns', 1, ''),
-                (u'Maintainable, easy to read and modify', 1, ''),
-            ]
-        ),
-
-        (
-            u'upstream',
-            u'Upstream Friendly',
-            [
-                (u'Follow upstream best practices', 1,
-                    u'Provide an option for a barebones "pure upstream" configuration'),
-                (u'Should go lock-step with deployment recommendations', 1,
-                    u'Provide tip-of-trunk testing if feasible'),
-                (u'Fresh charm on release day!', 1, ''),
-                (u"Endeavour to be upstream's recommended way to deploy that service in the cloud (website mention or something)", 1, ''),
-            ]
-        ),
-    ]
-
-    """Add the sample data into the db."""
-    for cat in initial:
-        category_dict = {
-            'name': cat[0],
-            'description': cat[1],
-            'questions': [{
-                'id': '{0}_{1}'.format(cat[0].lower(), i),
-                'description': q[0],
-                'points': q[1],
-                'extended_description': q[2]
-            } for i, q in enumerate(cat[2])]
-        }
-        db.qa.insert(category_dict)

=== removed file 'migrations/versions/002_Escape_dots_dollar_signs_percent_signs_in_mongo_keys.py'
--- migrations/versions/002_Escape_dots_dollar_signs_percent_signs_in_mongo_keys.py	2013-06-07 19:55:11 +0000
+++ migrations/versions/002_Escape_dots_dollar_signs_percent_signs_in_mongo_keys.py	1970-01-01 00:00:00 +0000
@@ -1,7 +0,0 @@
-# Copyright 2012, 2013 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# Escape dots, dollar signs, percent signs in mongo keys
-
-def upgrade(db, index_client):
-    """OBSOLETE; all instances are now migrated."""

=== removed file 'migrations/versions/003_remove_provider_qa_questions.py'
--- migrations/versions/003_remove_provider_qa_questions.py	2013-06-04 15:36:45 +0000
+++ migrations/versions/003_remove_provider_qa_questions.py	1970-01-01 00:00:00 +0000
@@ -1,42 +0,0 @@
-# Copyright 2013 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-"""
-Update the qa questions collection used by the qa form.
-
-Removes the providers questions from the reliable section because that
-information is automated.
-"""
-
-new_category_data = [
-    (
-        u'reliable',
-        u'Reliable',
-        [
-            (u'Check for integrity from upstream source', 1, u''),
-            (u'Fail gracefully if upstream source goes missing', 1, u''),
-            (u'Contain a suite of tests with the charm that pass', 1, u''),
-            (u'Passes tests from Jenkins on jujucharms.com', 1, u''),
-        ]
-    ),
-]
-
-
-def upgrade(db, index_client):
-    """Update each category of questions from new_category_data.
-
-    This update elaborates on the 001_* migration. Subsequent qa form
-    mirgrations can replace new_category_data and run this same method.
-    """
-    for cat in new_category_data:
-        name = cat[0]
-        category_dict = {
-            'name': name,
-            'description': cat[1],
-            'questions': [{
-                'id': '{0}_{1}'.format(cat[0].lower(), i),
-                'description': q[0],
-                'points': q[1],
-                'extended_description': q[2]
-            } for i, q in enumerate(cat[2])]
-        }
-        db.qa.update({'name': name}, category_dict)

=== removed file 'migrations/versions/004_remove_charm_errors.py'
--- migrations/versions/004_remove_charm_errors.py	2013-06-04 15:36:45 +0000
+++ migrations/versions/004_remove_charm_errors.py	1970-01-01 00:00:00 +0000
@@ -1,4 +0,0 @@
-# Copyright 2013 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-def upgrade(db, index_client):
-    db.drop_collection('charm-errors')

=== removed file 'migrations/versions/008_delete_icon_field.py'
--- migrations/versions/008_delete_icon_field.py	2013-06-11 19:13:07 +0000
+++ migrations/versions/008_delete_icon_field.py	1970-01-01 00:00:00 +0000
@@ -1,8 +0,0 @@
-# Copyright 2013 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-from charmworld.search import reindex
-
-
-def upgrade(db, index_client):
-    db.charms.update({}, {'$unset': {'icon': ''}}, multi=True)
-    reindex(index_client, list(db.charms.find({})))

=== removed file 'migrations/versions/009_add_downloads_attribute.py'
--- migrations/versions/009_add_downloads_attribute.py	2013-07-01 17:58:12 +0000
+++ migrations/versions/009_add_downloads_attribute.py	1970-01-01 00:00:00 +0000
@@ -1,21 +0,0 @@
-# Copyright 2013 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-from charmworld.search import reindex
-
-
-def upgrade(db, index_client):
-    """Add the downloads attribute to each charm.
-
-    Default to using their 30day acount and it'll be updated to be more
-    accurate on the next injest run.
-
-    """
-    charms = db.charms.find({})
-    for charm in charms:
-        if 'downloads_in_past_30_days' in charm:
-            charm['downloads'] = charm['downloads_in_past_30_days']
-        else:
-            charm['downloads'] = 0
-        db.charms.save(charm)
-
-    reindex(index_client, list(db.charms.find({})))

=== modified file 'migrations/versions/tests/test_migrations.py'
--- migrations/versions/tests/test_migrations.py	2013-07-15 21:31:05 +0000
+++ migrations/versions/tests/test_migrations.py	2013-07-22 20:55:30 +0000
@@ -26,31 +26,6 @@
             'comfigure_logging', true_configure_logging)
 
 
-class TestMigration004(MigrationTestBase):
-
-    def test_migration(self):
-        self.use_index_client()
-        self.db.create_collection('charm-errors')
-        self.versions.run_migration(self.db, self.index_client,
-                                    '004_remove_charm_errors.py')
-        self.assertNotIn('charm-errors', self.db.collection_names())
-
-
-class TestMigration008(MigrationTestBase):
-
-    def test_migration(self):
-        self.use_index_client()
-        source = CharmSource.from_request(self)
-        source.save({'_id': 'a', 'icon': 'asdf', 'asdf': 'asdf'})
-        source.save({'_id': 'b', 'icon': 'asdf', 'asdf': 'asdf'})
-        self.versions.run_migration(self.db, self.index_client,
-                                    '008_delete_icon_field.py')
-        for charm in source._get_all('a'):
-            self.assertNotIn('icon', charm)
-        for charm in source._get_all('b'):
-            self.assertNotIn('icon', charm)
-
-
 class TestMigration010(MigrationTestBase):
 
     def test_migration(self):