Merge lp:~elachuni/ubuntu-webcatalog/import-extras into lp:ubuntu-webcatalog

[Anthony Lenton]

Attempt to import icon information from extras.ubuntu.com whenever it's available.

This branch will attempt to import icon information from extras.ubuntu.com whenever it's available.

import_app_install_data will also pull in apps from extras, so that they start appearing within the Apps Directory. I'm still unsure as to how screenshot data should be retrieved, at the moment it'll only be imported if it's available from screenshots.ubuntu.com like other apps from the main repos (that is, screenshots won't be retrieved from MyApps for these apps).

[Danny Tamez (zematynnad)]

Hey Anthony - all looks fine.

review: Approve

=== modified file 'src/webcatalog/management/commands/import_app_install_data.py'
--- src/webcatalog/management/commands/import_app_install_data.py	2012-03-29 21:47:54 +0000
+++ src/webcatalog/management/commands/import_app_install_data.py	2012-04-03 17:55:23 +0000
@@ -22,15 +22,17 @@
     with_statement,
     )
 
+import json
 import os
-import json
+import re
 import shutil
 import tempfile
 import urllib
 
-from urlparse import urljoin
 from glob import iglob
 from optparse import make_option
+from tempfile import NamedTemporaryFile
+from urlparse import urljoin
 
 import apt
 from apt.cache import FetchFailedException
@@ -56,6 +58,7 @@
 deb http://archive.ubuntu.com/ubuntu {distroseries} main universe
 deb http://archive.ubuntu.com/ubuntu {distroseries}-updates main universe
 deb http://archive.canonical.com/ubuntu {distroseries} partner
+deb http://extras.ubuntu.com/ubuntu {distroseries} main
 """
 
     def __init__(self, distroseries):
@@ -175,7 +178,7 @@
 
                     self.output("Adding {0} to {1}\n".format(
                         screenshot_url,
-                        app.package_name), 1)
+                        app.package_name), 2)
                     app.applicationmedia_set.get_or_create(
                         media_type='screenshot',
                         url=screenshot_url,
@@ -248,11 +251,7 @@
                     break
 
         if icon_path:
-            parts = os.path.splitext(icon_path)
-            if parts[1] in ['.xpm', '.svg', '.tiff']:
-                target_path = parts[0] + '.png'
-                if create_png_from_file(target_path, icon_path, 32, 32):
-                    icon_path = target_path
+            icon_path = self.convert_to_png(icon_path)
             with open(icon_path, "r") as icon_file:
                 app.icon = ImageFile(icon_file)
                 app.save()
@@ -261,6 +260,34 @@
                         app.name, icon_path_generic,
                         '|'.join(supported_extensions)).encode('utf-8'), 1)
 
+    def convert_to_png(self, icon_path):
+        parts = os.path.splitext(icon_path)
+        if parts[1] in ['.xpm', '.svg', '.tiff']:
+            target_path = parts[0] + '.png'
+            if create_png_from_file(target_path, icon_path, 32, 32):
+                icon_path = target_path
+        return icon_path
+
+    def fetch_icon_from_extras(self, app, candidate):
+        """Fetch icon from the /meta/ directory on extras.ubuntu.com.
+
+        This method will only do anything if the provided app is in extras.
+        """
+        icon_name = candidate.record.get('Icon')
+        match = re.match('http://(.*)extras.ubuntu.com/', candidate.uri)
+        if icon_name and match:
+            uri = match.group() + 'meta/' + icon_name
+            response = urllib.urlopen(uri)
+            if response.getcode() == 200:
+                parts = os.path.splitext(icon_name)
+                with NamedTemporaryFile(suffix=parts[1]) as filep:
+                    filep.write(response.read())
+                    filep.flush()
+                    icon_path = self.convert_to_png(filep.name)
+                    with open(icon_path, "r") as icon_file:
+                        app.icon = ImageFile(icon_file)
+                        app.save()
+
     def update_from_cache(self, distroseries_name):
         distroseries, created = DistroSeries.objects.get_or_create(
             code_name=distroseries_name)
@@ -291,18 +318,16 @@
             version = apt.apt_pkg.upstream_version(candidate.version)
             if package.name in prefetched_apps:
                 app = prefetched_apps[package.name]
-                if (app.description == candidate.description and
-                    app.section == candidate.section and
-                    app.name == app_name and
-                    app.comment == app_comment):
-                        continue
-                app.description = candidate.description
-                app.section = candidate.section
-                app.version = version
-                app.name = app_name
-                app.comment = app_comment
-                app.save()
-
+                updated = False
+                for attribute, value in [
+                    ('description', candidate.description),
+                    ('section', candidate.section), ('version', version),
+                    ('name', app_name), ('comment', app_comment)]:
+                    if getattr(app, attribute) != value:
+                        setattr(app, attribute, value)
+                        updated = True
+                if updated:
+                    app.save()
             else:
                 app = Application.objects.create(
                     package_name=package.name,
@@ -314,6 +339,7 @@
                     comment=app_comment,
                     )
             self.get_screenshot_urls_from_server(app)
+            self.fetch_icon_from_extras(app, candidate)
         if self.verbosity == 1:
             self.output("\n", 1)
         return distroseries

=== modified file 'src/webcatalog/tests/test_commands.py'
--- src/webcatalog/tests/test_commands.py	2012-03-29 21:47:54 +0000
+++ src/webcatalog/tests/test_commands.py	2012-04-03 17:55:23 +0000
@@ -363,6 +363,7 @@
 deb http://archive.ubuntu.com/ubuntu natty main universe
 deb http://archive.ubuntu.com/ubuntu natty-updates main universe
 deb http://archive.canonical.com/ubuntu natty partner
+deb http://extras.ubuntu.com/ubuntu natty main
 """,
             sources_list_content)
         if self.use_mock_apt_cache:
@@ -522,6 +523,49 @@
         qs = app.applicationmedia_set.filter(media_type='screenshot')
         self.assertEqual(2, qs.count())
 
+    @patch('urllib.urlopen')
+    def test_fetch_icon_from_extras(self, mock_urlopen):
+        mock_urlopen.return_value.getcode.return_value = 200
+        mock_urlopen.return_value.read.return_value = "I'm a png!"
+        app = self.factory.make_application()
+        candidate = Mock()
+        candidate.uri = 'http://extras.ubuntu.com/pool/f/foobar.deb'
+        candidate.record = {'Icon': 'foo.png'}
+        command = ImportAppInstallCommand()
+
+        command.fetch_icon_from_extras(app, candidate)
+
+        self.assertEqual("I'm a png!", app.icon.read())
+        self.assertEqual(1, mock_urlopen.call_count)
+        mock_urlopen.assert_called_with(
+            'http://extras.ubuntu.com/meta/foo.png')
+
+    @patch('urllib.urlopen')
+    def test_fetches_nothing_if_no_icon(self, mock_urlopen):
+        app = self.factory.make_application()
+        candidate = Mock()
+        candidate.uri = 'http://extras.ubuntu.com/pool/f/foobar.deb'
+        candidate.record = {}
+        command = ImportAppInstallCommand()
+
+        command.fetch_icon_from_extras(app, candidate)
+
+        self.assertEqual(0, mock_urlopen.call_count)
+        self.assertFalse(app.icon)
+
+    @patch('urllib.urlopen')
+    def test_fetches_nothing_if_not_in_extras(self, mock_urlopen):
+        app = self.factory.make_application()
+        candidate = Mock()
+        candidate.uri = 'http://archive.ubuntu.com/pool/f/foobar.deb'
+        candidate.record = {'Icon': 'foobar.png'}
+        command = ImportAppInstallCommand()
+
+        command.fetch_icon_from_extras(app, candidate)
+
+        self.assertEqual(0, mock_urlopen.call_count)
+        self.assertFalse(app.icon)
+
 
 class ImportRatingsTestCase(TestCaseWithFactory):