Merge lp:~snappy-dev/snapcraft/more-apt into lp:~snappy-dev/snapcraft/core

[Sergio Schvezov]

Improving stage-package handling

[lp:~snappy-dev/snapcraft/more-apt]

190. By Sergio Schvezov on 2015-09-21

Remove tests that doesn't make sense as a unit test

191. By Sergio Schvezov on 2015-09-21

merge trunk

[Michael Vogt (mvo)]

Good stuff!

review: Approve

=== modified file 'snapcraft/__init__.py'
--- snapcraft/__init__.py	2015-09-17 14:14:14 +0000
+++ snapcraft/__init__.py	2015-09-21 15:29:20 +0000
@@ -31,6 +31,10 @@
     def PLUGIN_STAGE_PACKAGES(self):
         return getattr(self, '_PLUGIN_STAGE_PACKAGES', [])
 
+    @property
+    def PLUGIN_STAGE_SOURCES(self):
+        return getattr(self, '_PLUGIN_STAGE_SOURCES', [])
+
     def __init__(self, name, options):
         self.name = name
         self.options = options
@@ -99,9 +103,9 @@
         os.makedirs(d, exist_ok=True)
 
     def setup_stage_packages(self):
-        ubuntu = snapcraft.repo.Ubuntu(self.ubuntudir)
         part_stage_packages = getattr(self.options, 'stage_packages', []) or []
         if self.PLUGIN_STAGE_PACKAGES or part_stage_packages:
+            ubuntu = snapcraft.repo.Ubuntu(self.ubuntudir, sources=self.PLUGIN_STAGE_SOURCES)
             ubuntu.get(self.PLUGIN_STAGE_PACKAGES + part_stage_packages)
             ubuntu.unpack(self.installdir)
 
 
=== modified file 'snapcraft/manifest.txt'
--- snapcraft/manifest.txt	2015-09-01 19:04:49 +0000
+++ snapcraft/manifest.txt	2015-09-21 15:29:20 +0000
@@ -1,5 +1,6 @@
 adduser
 apt
+apt-utils
 base-files
 base-passwd
 bash
@@ -7,12 +8,14 @@
 coreutils
 dash
 debconf
+debconf-i18n
 debianutils
 diffutils
 dmsetup
 dpkg
 e2fslibs
 e2fsprogs
+file
 findutils
 gcc-4.9-base
 gcc-5-base
@@ -26,6 +29,8 @@
 insserv
 libacl1
 libapparmor1
+libapt
+libapt-inst1.5
 libapt-pkg4.12
 libattr1
 libaudit-common
@@ -44,8 +49,11 @@
 libgcc1
 libgcrypt20
 libgpg-error0
+libgpm2
 libkmod2
+liblocale-gettext-perl
 liblzma5
+libmagic1
 libmount1
 libncurses5
 libncursesw5
@@ -65,6 +73,9 @@
 libss2
 libstdc++6
 libsystemd0
+libtext-charwidth-perl
+libtext-iconv-perl
+libtext-wrapi18n-perl
 libtinfo5
 libudev1
 libusb-0.1-4
@@ -74,6 +85,8 @@
 login
 lsb-base
 makedev
+manpages
+manpages-dev
 mawk
 mount
 multiarch-support
 
=== modified file 'snapcraft/repo.py'
--- snapcraft/repo.py	2015-09-07 20:23:57 +0000
+++ snapcraft/repo.py	2015-09-21 15:29:20 +0000
@@ -14,11 +14,28 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import apt
 import glob
 import itertools
 import os
+import string
 import subprocess
+import urllib
+import urllib.request
+
+import apt
+from xml.etree import ElementTree
+
+_DEFAULT_SOURCES = '''deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid main restricted
+deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid-updates main restricted
+deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid universe
+deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid-updates universe
+deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid multiverse
+deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid-updates multiverse
+deb http://security.ubuntu.com/ubuntu vivid-security main restricted
+deb http://security.ubuntu.com/ubuntu vivid-security universe
+deb http://security.ubuntu.com/ubuntu vivid-security multiverse
+'''
+_GEOIP_SERVER = "http://geoip.ubuntu.com/lookup"
 
 
 class PackageNotFoundError(Exception):
@@ -43,33 +60,47 @@
 
 class Ubuntu:
 
-    def __init__(self, download_dir, recommends=False):
-        self.apt_cache = apt.Cache()
-        self.manifest_dep_names = self._manifest_dep_names()
+    def __init__(self, rootdir, recommends=False, sources=_DEFAULT_SOURCES):
+        sources = sources or _DEFAULT_SOURCES
+        self.downloaddir = os.path.join(rootdir, 'download')
+        self.rootdir = rootdir
+        self.apt_cache = _setup_apt_cache(rootdir, sources)
         self.recommends = recommends
-        self.download_dir = download_dir
 
     def get(self, package_names):
-        # TODO cleanup download_dir for clean gets and unpacks
-        self.all_dep_names = set()
-
-        all_package_names = self._compute_deps(package_names)
-
-        for pkg in all_package_names:
-            self.apt_cache[pkg].candidate.fetch_binary(destdir=self.download_dir)
-
-        return all_package_names
-
-    def unpack(self, root_dir):
-        pkgs_abs_path = glob.glob(os.path.join(self.download_dir, '*.deb'))
+        os.makedirs(self.downloaddir, exist_ok=True)
+
+        manifest_dep_names = self._manifest_dep_names()
+
+        for name in package_names:
+            try:
+                self.apt_cache[name].mark_install()
+            except KeyError:
+                raise PackageNotFoundError(name)
+
+        for pkg in self.apt_cache:
+            # those should be already on each system, it also prevents
+            # diving into downloading libc6
+            if (pkg.candidate.priority in 'essential' and
+               pkg.name not in package_names):
+                print('Skipping priority essential/imporant %s' % pkg.name)
+                continue
+            if (pkg.name in manifest_dep_names and pkg.name not in package_names):
+                print('Skipping blacklisted from manifest package %s' % pkg.name)
+                continue
+            if pkg.marked_install:
+                pkg.candidate.fetch_binary(destdir=self.downloaddir)
+
+    def unpack(self, rootdir):
+        pkgs_abs_path = glob.glob(os.path.join(self.downloaddir, '*.deb'))
         for pkg in pkgs_abs_path:
             # TODO needs elegance and error control
             try:
-                subprocess.check_call(['dpkg-deb', '--extract', pkg, root_dir])
+                subprocess.check_call(['dpkg-deb', '--extract', pkg, rootdir])
             except subprocess.CalledProcessError:
                 raise UnpackError(pkg)
 
-        _fix_symlinks(root_dir)
+        _fix_symlinks(rootdir)
 
     def _manifest_dep_names(self):
         manifest_dep_names = set()
@@ -82,36 +113,35 @@
 
         return manifest_dep_names
 
-    def _compute_deps(self, package_names):
-        self._add_deps(package_names)
-
-        for pkg in package_names:
-            if pkg not in self.all_dep_names:
-                raise PackageNotFoundError(pkg)
-
-        return sorted(self.all_dep_names)
-
-    def _add_deps(self, package_names):
-        def add_deps(packages):
-            for pkg in packages:
-                # Remove the :any in packages
-                # TODO support multiarch
-                pkg = pkg.rsplit(':', 1)[0]
-                if pkg in self.all_dep_names:
-                    continue
-                if pkg in self.manifest_dep_names and pkg not in package_names:
-                    continue
-                deps = set()
-                try:
-                    candidate_pkg = self.apt_cache[pkg].candidate
-                except KeyError:
-                    raise PackageNotFoundError(pkg)
-                deps = candidate_pkg.dependencies
-                if self.recommends:
-                    deps += candidate_pkg.recommends
-                self.all_dep_names.add(pkg)
-                add_deps([x[0].name for x in deps])
-        add_deps(package_names)
+
+def get_geoip_country_code_prefix():
+    try:
+        with urllib.request.urlopen(_GEOIP_SERVER) as f:
+            xml_data = f.read()
+        et = ElementTree.fromstring(xml_data)
+        cc = et.find("CountryCode")
+        if cc is None:
+            return ""
+        return cc.text.lower() + "."
+    except (ElementTree.ParseError, urllib.error.URLError):
+        pass
+    return ""
+
+
+def _setup_apt_cache(rootdir, sources):
+    os.makedirs(os.path.join(rootdir, 'etc', 'apt'), exist_ok=True)
+    srcfile = os.path.join(rootdir, 'etc', 'apt', 'sources.list')
+    with open(srcfile, 'w') as f:
+        mirror_prefix = get_geoip_country_code_prefix()
+        sources_list = string.Template(sources).substitute(
+            {"mirror": mirror_prefix})
+        f.write(sources_list)
+    progress = apt.progress.text.AcquireProgress()
+    apt_cache = apt.Cache(rootdir=rootdir, memonly=True)
+    apt_cache.update(fetch_progress=progress, sources_list=srcfile)
+    apt_cache.open()
+
+    return apt_cache
 
 
 def _fix_symlinks(debdir):
 
=== modified file 'snapcraft/tests/test_repo.py'
--- snapcraft/tests/test_repo.py	2015-09-01 19:04:49 +0000
+++ snapcraft/tests/test_repo.py	2015-09-21 15:29:20 +0000
@@ -23,15 +23,6 @@
 
 class UbuntuTestCase(tests.TestCase):
 
-    def test_unrecognized_package_raises_exception(self):
-        ubuntu = repo.Ubuntu('download_dir')
-
-        with self.assertRaises(repo.PackageNotFoundError) as raised:
-            ubuntu.get(['test_package'])
-
-        expected_message = 'The Ubuntu package \'test_package\' was not found'
-        self.assertEqual(raised.exception.message, expected_message)
-
     def test_fix_symlinks(self):
         tempdirObj = tempfile.TemporaryDirectory()
         self.addCleanup(tempdirObj.cleanup)