Merge lp:~abentley/charmworld/stop-ingesting-icon into lp:~juju-jitsu/charmworld/trunk

[Aaron Bentley]

Stop adding icon during ingest.

The 006 migration deleted icons from the charm data, but ingest restored them.

This branch updates ingest to stop storing icon data and renames the 006 migration to 007 so that it will run again.

[Curtis Hovey (sinzui)]

Thank you.

review: Approve (code)

=== modified file 'charmworld/jobs/ingest.py'
--- charmworld/jobs/ingest.py	2013-05-29 20:43:01 +0000
+++ charmworld/jobs/ingest.py	2013-06-05 18:36:28 +0000
@@ -1,7 +1,6 @@
 # Copyright 2013 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
-from base64 import b64encode
 import calendar
 import contextlib
 from datetime import (
@@ -121,17 +120,6 @@
             (quote_key(cfile.filename), dict(cfile)) for cfile in
             self.store_branch_files(charm_data)
         ])
-        if ICON_FILENAME in os.listdir(charm_data['branch_dir']):
-            file_path = os.path.join(charm_data['branch_dir'], ICON_FILENAME)
-            with open(file_path, 'rb') as icon_file:
-                charm_data['icon'] = b64encode(icon_file.read())
-        else:
-            # It might happen that an existing icon is removed from the
-            # charm. We should assume that this is a deliberate decision
-            # by the charm's maintainer, be it for an aesthetic, legal
-            # or any other reason.
-            if 'icon' in charm_data:
-                del charm_data['icon']
         return charm_data
 
     def checkout_charm(self, charm_data, branch_dir):

=== modified file 'charmworld/jobs/lp.py'
--- charmworld/jobs/lp.py	2013-05-24 09:42:29 +0000
+++ charmworld/jobs/lp.py	2013-06-05 18:36:28 +0000
@@ -20,7 +20,12 @@
 from utils import parse_branch
 
 
-def available_charms(charm_data=None, limit=None):
+default = object()
+
+
+def available_charms(charm_data=None, limit=None, import_filter=default):
+    if import_filter is default:
+        import_filter = CHARM_IMPORT_FILTER
     log = logging.getLogger("charm.launchpad")
     if not charm_data:
         charm_data = get_branch_tips()
@@ -36,8 +41,8 @@
             log.warning("Unable to parse repo %s", repo)
             continue
 
-        if CHARM_IMPORT_FILTER and not data['branch_spec'].startswith(
-                CHARM_IMPORT_FILTER):
+        if import_filter and not data['branch_spec'].startswith(
+                import_filter):
             continue
 
         if data["bname"] not in ("trunk", "trunk-1"):
@@ -59,9 +64,9 @@
             break
 
 
-def queue_charms(out_queue, limit=None):
+def queue_charms(out_queue, limit=None, import_filter=default):
     log = logging.getLogger("charm.launchpad")
-    for charm in available_charms(limit=limit):
+    for charm in available_charms(limit=limit, import_filter=import_filter):
         added = out_queue.put(charm)
         if added and 0:
             log.info("Queued %s", charm)
@@ -74,6 +79,10 @@
         "--limit",
         help="Maximum number of charms to queue.",
         type=int)
+    parser.add_argument(
+        "--prefix",
+        help="Only charms matching this prefix are queued.",
+        type=str, default=default)
     args = parser.parse_args()
     log = logging.getLogger("charm.launchpad")
     settings = get_ini()
@@ -89,7 +98,7 @@
         with lock('ingest-queue', int(settings['script_lease_time']) * 60,
                   db, log):
             queue = get_queue(CHARM_QUEUE)
-            queue_charms(queue, charm_import_limit)
+            queue_charms(queue, charm_import_limit, args.prefix)
     except LockHeld, error:
         log.warn(str(error))
 

=== modified file 'charmworld/jobs/tests/test_bzr.py'
--- charmworld/jobs/tests/test_bzr.py	2013-04-24 20:30:58 +0000
+++ charmworld/jobs/tests/test_bzr.py	2013-06-05 18:36:28 +0000
@@ -102,7 +102,7 @@
         job.setup(db=self.db)
         job.add_files(charm_data)
         self.assertIn('files', charm_data)
-        self.assertIn('icon', charm_data)
+        self.assertNotIn('icon', charm_data)
 
 
 @contextmanager

=== modified file 'charmworld/jobs/tests/test_lp.py'
--- charmworld/jobs/tests/test_lp.py	2013-05-23 10:57:51 +0000
+++ charmworld/jobs/tests/test_lp.py	2013-06-05 18:36:28 +0000
@@ -17,7 +17,7 @@
 from charmworld.jobs.utils import get_queue
 
 
-def available_charms_mock(charm_data=None, limit=None):
+def available_charms_mock(charm_data=None, limit=None, import_filter=None):
     yield {'branch_dir': 'foo', 'branch_spec': 'bar'}
 
 
@@ -89,6 +89,21 @@
             charms = [charm for charm in available_charms(charm_data)]
         self.assertEqual([], charms)
 
+    def test_available_charms_explicit_filter(self):
+        # If the branch doesn't start with the defined filter (i.e.
+        # ~charmers/charms/precise) it is not returned
+        charm_data = [[u'~not-charmers/charms/precise/someproject/trunk',
+                       u'[e-mail]-20120329093714-s2m9e28dwotmijqc',
+                       []],
+                      [u'~charmers/charms/precise/someproject/trunk',
+                       u'[e-mail]-20120329093714-s2m9e28dwotmijqc',
+                       []],
+                      ]
+        charms = available_charms(
+            charm_data, import_filter='~charmers/charms/precise')
+        self.assertEqual(['~charmers/charms/precise/someproject/trunk'],
+                         [charm['branch_spec'] for charm in charms])
+
     def test_available_charms_not_trunk(self):
         # If the branch is not trunk or trunk-1, the charm is not returned by
         # available charms.

=== added file 'migrations/versions/006_no_op.py'
--- migrations/versions/006_no_op.py	1970-01-01 00:00:00 +0000
+++ migrations/versions/006_no_op.py	2013-06-05 18:36:28 +0000
@@ -0,0 +1,5 @@
+# Copyright 2013 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+def upgrade(db, index_client):
+    """006 lacked a corresponding ingest update, so now disabled."""

=== renamed file 'migrations/versions/006_delete_icon_field.py' => 'migrations/versions/007_delete_icon_field.py'
=== modified file 'migrations/versions/tests/test_migrations.py'
--- migrations/versions/tests/test_migrations.py	2013-06-04 18:22:19 +0000
+++ migrations/versions/tests/test_migrations.py	2013-06-05 18:36:28 +0000
@@ -66,14 +66,14 @@
         self.assertNotIn('charm-errors', self.db.collection_names())
 
 
-class TestMigration006(MongoTestBase):
+class TestMigration007(MongoTestBase):
 
     def test_migration(self):
         self.use_index_client()
         source = CharmSource.from_request(self)
         source.save({'_id': 'a', 'icon': 'asdf', 'asdf': 'asdf'})
         source.save({'_id': 'b', 'icon': 'asdf', 'asdf': 'asdf'})
-        run_migration(self.db, self.index_client, '006_delete_icon_field.py')
+        run_migration(self.db, self.index_client, '007_delete_icon_field.py')
         for charm in source._get_all('a'):
             self.assertNotIn('icon', charm)
         for charm in source._get_all('b'):