Merge lp:~michael.nelson/ubuntu-webcatalog/determine-app-install-version into lp:ubuntu-webcatalog

[Michael Nelson]

[r=lukasz] Determine most recent app-install-data uri for download for given distroseries.

Overview
========
This branch enables the importer to find the latest app install data for a given distroseries, removing the hard-coded natty package uri.

I was unsure whether to use mocks in the test - it works without the mocks, just takes a while.

Example runs for natty and lucid

michael@devmav:~/isd/webcat/current_work$ fab manage:'import_app_install_data natty'
[localhost] run: virtualenv/bin/python django_project/manage.py import_app_install_data natty
Determining URI for current version of app-install-data in natty.
creating: /tmp/tmpJieAlT/var/lib/dpkg
creating: /tmp/tmpJieAlT/var/cache/apt/archives/partial
creating: /tmp/tmpJieAlT/var/lib/apt/lists/partial
Downloading app-install-data-ubuntu from http://archive.ubuntu.com/ubuntu/pool/main/a/app-install-data-ubuntu/app-install-data_0.11.04.7.1_all.deb...Done.
Processing application data to update database...
No icon found for Eclipse at /tmp/tmpotEAQf/usr/share/app-install/icons/eclipse.(xpm|png|svg|tiff).
Eclipse created.
No icon found for Mah-Jong at /tmp/tmpotEAQf/usr/share/app-install/icons/mah-jong.(xpm|png|svg|tiff).
Mah-Jong created.
No icon found for Personal File Sharing at /tmp/tmpotEAQf/usr/share/app-install/icons/folder-remote.(xpm|png|svg|tiff).
Personal File Sharing created.
No icon found for Advanced Strategic Command at /tmp/tmpotEAQf/usr/share/app-install/icons/asc.(xpm|png|svg|tiff).
Advanced Strategic Command created.
Tangerine Media Sharing created.
^C
Stopped.
michael@devmav:~/isd/webcat/current_work$ fab manage:'import_app_install_data lucid'
[localhost] run: virtualenv/bin/python django_project/manage.py import_app_install_data lucid
Determining URI for current version of app-install-data in lucid.
creating: /tmp/tmpxLr0I8/var/lib/dpkg
creating: /tmp/tmpxLr0I8/var/cache/apt/archives/partial
creating: /tmp/tmpxLr0I8/var/lib/apt/lists/partial
Downloading app-install-data-ubuntu from http://archive.ubuntu.com/ubuntu/pool/main/a/app-install-data-ubuntu/app-install-data_0.10.04.7_all.deb...Done.
Processing application data to update database...
Granatier created.
No icon found for Eclipse at /tmp/tmpBzKAd8/usr/share/app-install/icons/eclipse.(xpm|png|svg|tiff).
Eclipse created.
No icon found for Mah-Jong at /tmp/tmpBzKAd8/usr/share/app-install/icons/mah-jong.(xpm|png|svg|tiff).
Mah-Jong created.
No icon found for KThesaurus at /tmp/tmpBzKAd8/usr/share/app-install/icons/kthesaurus.(xpm|png|svg|tiff).
KThesaurus created.
No icon found for Personal File Sharing at /tmp/tmpBzKAd8/usr/share/app-install/icons/folder-remote.(xpm|png|svg|tiff).
Personal File Sharing created.
No icon found for Advanced Strategic Command at /tmp/tmpBzKAd8/usr/share/app-install/icons/asc.(xpm|png|svg|tiff).
Advanced Strategic Command created.

[lp:~michael.nelson/ubuntu-webcatalog/determine-app-install-version]

13. By Michael Nelson on 2011-04-14

stdout tweaks.

14. By Michael Nelson on 2011-04-15

Ensure we don't call apt.cache during get_latest_app_data_for_series.

15. By Michael Nelson on 2011-04-15

Removed verbosity as a kwarg for all methods, instead using a class attribute.

=== modified file 'src/webcatalog/management/commands/import_app_install_data.py'
--- src/webcatalog/management/commands/import_app_install_data.py	2011-04-13 11:09:59 +0000
+++ src/webcatalog/management/commands/import_app_install_data.py	2011-04-15 09:19:27 +0000
@@ -26,13 +26,14 @@
 import shutil
 import tempfile
 import urllib
-from apt_inst import DebFile
 from glob import iglob
 from optparse import make_option
 from StringIO import StringIO
 
+import apt
+from apt_inst import DebFile
 from django.core.files.images import ImageFile
-from django.core.management.base import BaseCommand
+from django.core.management.base import LabelCommand
 
 from webcatalog.forms import ApplicationForm
 
@@ -41,34 +42,26 @@
     ]
 
 
-class Command(BaseCommand):
+class Command(LabelCommand):
 
     help = "Update Application data from app-install-data package."
-    option_list = BaseCommand.option_list + (
+    option_list = LabelCommand.option_list + (
         make_option('--local-app-install-deb',
             action='store',
             dest='local_app_install_deb',
             default='',
             help=('Use a local app-install-data deb package rather than '
                   'downloading from the archive.')),
-        )
+            )
+    verbosity = 0
 
-    def handle(self, *args, **options):
+    def handle_label(self, distroseries,  **options):
         self.verbosity = int(options['verbosity'])
+        data_dir = tempfile.mkdtemp()
         # Download app install data when requested.
         if options['local_app_install_deb'] == '':
-            install_data_url = ("http://archive.ubuntu.com/"
-                                "ubuntu/pool/main/a/app-install-data-ubuntu/"
-                                "app-install-data_0.11.04.7.1_all.deb")
-            if self.verbosity > 0:
-                self.stdout.write(
-                    u"Downloading app-install-data-ubuntu from {0}...".format(
-                        install_data_url))
-            tmp_file = tempfile.NamedTemporaryFile()
-            urllib.urlretrieve(install_data_url, tmp_file.name)
-            deb_location = tmp_file.name
-            if self.verbosity > 0:
-                self.stdout.write("Done.\n")
+            deb_location = self.get_latest_app_data_for_series(
+                distroseries, working_dir=data_dir)
         else:
             deb_location = options['local_app_install_deb']
 
@@ -77,7 +70,6 @@
         if self.verbosity > 0:
             self.stdout.write(
                 "Processing application data to update database...\n")
-        data_dir = tempfile.mkdtemp()
         deb_file.data.extractall(data_dir)
         matcher = data_dir + '/usr/share/app-install/desktop/*.desktop'
         icon_dir = data_dir + '/usr/share/app-install/icons/'
@@ -86,10 +78,48 @@
 
         shutil.rmtree(data_dir)
 
-    def get_uri_for_series(self, distro_series):
+    def get_latest_app_data_for_series(self, distroseries, working_dir):
         # Write a temporary sources.list, update the cache and access:
         # cache['app-install-data-ubuntu'].candidate.uri
-        pass
+        install_data_url = self.get_app_data_uri_for_series(distroseries)
+        rest, sep, deb_filename = install_data_url.rpartition('/')
+
+        if self.verbosity > 0:
+            self.stdout.write(
+                u"Downloading app-install-data-ubuntu from {0}...".format(
+                    install_data_url))
+        deb_location = os.path.join(working_dir, deb_filename)
+        urllib.urlretrieve(install_data_url, deb_location)
+        if self.verbosity > 0:
+            self.stdout.write("Done.\n")
+        return deb_location
+
+    def get_app_data_uri_for_series(self, distroseries):
+        # Create a temporary apt cache to find the uri for the
+        # current version in the given series.
+        if self.verbosity > 0:
+            self.stdout.write(
+                "Determining URI for current version of "
+                "app-install-data in {0}.\n".format(distroseries))
+
+        # First we create an appropriate temporary sources.list entry.
+        tmp_sources_dir = tempfile.mkdtemp()
+        os.makedirs(os.path.join(tmp_sources_dir, 'etc', 'apt'))
+        tmp_sources_list = os.path.join(
+            tmp_sources_dir, 'etc', 'apt', 'sources.list')
+        sources_list = open(tmp_sources_list, 'w+')
+        sources_list.write(
+            'deb http://archive.ubuntu.com/ubuntu {0} main\n'.format(
+                distroseries))
+        sources_list.close()
+
+        cache = apt.Cache(rootdir=tmp_sources_dir, memonly=True)
+        cache.update(sources_list=tmp_sources_list)
+        cache.open()
+        uri = cache['app-install-data'].candidate.uri
+
+        shutil.rmtree(tmp_sources_dir)
+        return uri
 
     def process_desktop_file(self, desktop_file_path, icon_dir):
         with open(desktop_file_path, 'r') as desktop_file:

=== modified file 'src/webcatalog/tests/test_commands.py'
--- src/webcatalog/tests/test_commands.py	2011-04-13 17:19:16 +0000
+++ src/webcatalog/tests/test_commands.py	2011-04-15 09:19:27 +0000
@@ -26,7 +26,11 @@
 import tempfile
 
 from django.core.management import call_command
-from mock import patch
+from mock import (
+    patch,
+    MagicMock,
+    Mock,
+    )
 
 from webcatalog.models import Application
 from webcatalog.management.commands.import_app_install_data import Command
@@ -41,24 +45,36 @@
 class ImportAppInstallTestCase(TestCaseWithFactory):
 
     def test_app_data_downloaded_by_default(self):
+        get_data_fn = ('webcatalog.management.commands.'
+                       'import_app_install_data.Command.'
+                       'get_latest_app_data_for_series')
+        with patch(get_data_fn) as mock_get_data_fn:
+            mock_get_data_fn.return_value = self.factory.get_test_path(
+                'app-install-data-test_all.deb')
+            call_command('import_app_install_data', 'onion', verbosity=0)
+
+        self.assertEqual(1, mock_get_data_fn.call_count)
+        self.assertEqual(('onion', ), mock_get_data_fn.call_args[0])
+
+    def test_get_latest_app_data_for_series(self):
         # Requested and downloaded from the server.
-        with patch('urllib.urlretrieve') as mock_urlretrieve:
-            with patch('tempfile.NamedTemporaryFile') as mock_named_tmp_file:
-                mock_tmp_file = mock_named_tmp_file.return_value
-                mock_tmp_file.name = self.factory.get_test_path(
-                    'app-install-data-test_all.deb')
-                call_command('import_app_install_data', verbosity=0)
+        tmp_dir = tempfile.mkdtemp()
+        get_uri_fn = (
+            'webcatalog.management.commands.import_app_install_data.'
+            'Command.get_app_data_uri_for_series')
+        with patch(get_uri_fn) as mock_get_uri:
+            mock_get_uri.return_value = 'http://example.com/my.deb'
+            with patch('urllib.urlretrieve') as mock_urlretrieve:
+                Command().get_latest_app_data_for_series('natty', tmp_dir)
+        shutil.rmtree(tmp_dir)
 
         mock_urlretrieve.assert_called_with(
-            ("http://archive.ubuntu.com/"
-             "ubuntu/pool/main/a/app-install-data-ubuntu/"
-             "app-install-data_0.11.04.7.1_all.deb"),
-             self.factory.get_test_path('app-install-data-test_all.deb'))
+            'http://example.com/my.deb', os.path.join(tmp_dir, 'my.deb'))
 
     def test_local_app_install_deb(self):
         # We don't hit the network when provided a local app-install deb instead.
         with patch('urllib.urlretrieve') as mock_urlretrieve:
-            call_command('import_app_install_data',
+            call_command('import_app_install_data', 'natty',
                 local_app_install_deb=self.factory.get_test_path(
                     'app-install-data-test_all.deb'),
                 verbosity=0)
@@ -69,7 +85,7 @@
         self.assertEqual(0, Application.objects.count())
 
         call_command(
-            'import_app_install_data',
+            'import_app_install_data', 'natty',
             local_app_install_deb=self.factory.get_test_path(
                 'app-install-data-test_all.deb'),
             verbosity=0)
@@ -83,7 +99,7 @@
 
     def test_icon_added_to_model(self):
         call_command(
-            'import_app_install_data',
+            'import_app_install_data', 'natty',
             local_app_install_deb=self.factory.get_test_path(
                 'app-install-data-test_all.deb'),
             verbosity=0)
@@ -97,3 +113,19 @@
         self.assertEqual(
             os.path.getsize(self.factory.get_test_path('scribus.xpm')),
             scribus.icon.size)
+
+    def test_get_app_data_uri_for_series(self):
+        # Note: you can test this with the real cache by commenting out
+        # the mocks - it just takes 4-5 seconds or so.
+        apt_cache = (
+            'webcatalog.management.commands.import_app_install_data.'
+            'apt.Cache')
+        with patch(apt_cache) as mock_apt_cache_cls:
+            mock_cache_item = Mock()
+            mock_cache_item.candidate.uri = 'http://example.com/my.deb'
+            mock_cache = MagicMock()
+            mock_cache.__getitem__.return_value = mock_cache_item
+            mock_apt_cache_cls.return_value = mock_cache
+            uri = Command().get_app_data_uri_for_series('natty')
+
+        self.assertEqual('http://example.com/my.deb', uri)