Merge lp:~ev/apt-clone/739489 into lp:apt-clone

[Evan]

See bug 739489 for the details.

[Evan (ev)]

I was curious if we should instead assume a 'release-pocket' format, rather than hard code backports, updates, etc in.

=== modified file 'apt_clone.py'
--- apt_clone.py	2011-03-31 11:26:40 +0000
+++ apt_clone.py	2011-04-06 14:12:27 +0000
@@ -17,6 +17,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
 import apt
+from apt.cache import FetchFailedException
 import apt_pkg
 import logging
 import glob
@@ -69,6 +70,11 @@
         ret = subprocess.call(["debootstrap", distro, targetdir])
         return (ret == 0)
 
+    def merge_keys(self, fromkeyfile, intokeyfile):
+        ret = subprocess.call(['apt-key', '--keyring', intokeyfile,
+                               'add', fromkeyfile])
+        return (ret == 0)
+
 class AptClone(object):
     """ clone the package selection/installation of a existing system
         using the information that apt provides
@@ -298,7 +304,12 @@
         if new_distro:
             self._rewrite_sources_list(target, new_distro)
         cache = self._cache_cls(rootdir=target)
-        cache.update(apt.progress.base.AcquireProgress())
+        try:
+            cache.update(apt.progress.base.AcquireProgress())
+        except FetchFailedException:
+            # This cannot be resolved here, but it should not be interpreted as
+            # a fatal error.
+            pass
         cache.open()
         # try to replay cache and see thats missing
         missing = self._restore_package_selection_in_cache(statefile, cache)
@@ -306,6 +317,9 @@
 
     def _restore_sources_list(self, statefile, targetdir):
         tar = tarfile.open(statefile)
+        existing = os.path.join(targetdir, "etc", "apt", "sources.list")
+        if os.path.exists(existing):
+            shutil.copy(existing, '%s.apt-clone' % existing)
         tar.extract("./etc/apt/sources.list", targetdir)
         try:
             tar.extract("./etc/apt/sources.list.d", targetdir)
@@ -313,6 +327,10 @@
             pass
 
     def _restore_apt_keyring(self, statefile, targetdir):
+        existing = os.path.join(targetdir, "etc", "apt", "trusted.gpg")
+        backup = '%s.apt-clone' % existing
+        if os.path.exists(existing):
+            shutil.copy(existing, backup)
         tar = tarfile.open(statefile)
         try:
             tar.extract("./etc/apt/trusted.gpg", targetdir)
@@ -322,6 +340,9 @@
             tar.extract("./etc/apt/trusted.gpg.d", targetdir)
         except KeyError:
             pass
+        if os.path.exists(backup):
+            self.commands.merge_keys(backup, existing)
+            os.remove(backup)
 
     def _restore_package_selection_in_cache(self, statefile, cache):
         # reinstall packages
@@ -366,7 +387,12 @@
     def _restore_package_selection(self, statefile, targetdir):
         # create new cache
         cache = self._cache_cls(rootdir=targetdir)
-        cache.update(self.fetch_progress)
+        try:
+            cache.update(self.fetch_progress)
+        except FetchFailedException:
+            # This cannot be resolved here, but it should not be interpreted as
+            # a fatal error.
+            pass
         cache.open()
         self._restore_package_selection_in_cache(statefile, cache)
         # do it
@@ -386,7 +412,7 @@
         self.commands.install_debs(debs, targetdir)
 
     def _rewrite_sources_list(self, targetdir, new_distro):
-        from aptsources.sourceslist import SourcesList
+        from aptsources.sourceslist import SourcesList, SourceEntry
         apt_pkg.config.set(
             "Dir::Etc::sourcelist",
             os.path.abspath(os.path.join(targetdir, "etc", "apt", "sources.list")))
@@ -394,10 +420,36 @@
             "Dir::Etc::sourceparts",
             os.path.abspath(os.path.join(targetdir, "etc", "apt", "sources.list.d")))
         sources = SourcesList()
+
         for entry in sources.list[:]:
             if entry.invalid or entry.disabled:
                 continue
-            entry.dist = new_distro
+            replacement = ''
+            for pocket in ('updates', 'security', 'backports'):
+                if entry.dist.endswith('-%s' % pocket):
+                    replacement = '%s-%s' % (new_distro, pocket)
+                    break
+            if replacement:
+                entry.dist = replacement
+            else:
+                entry.dist = new_distro
+
+        existing = os.path.join(targetdir, "etc", "apt",
+                                "sources.list.apt-clone")
+        sourcelist = apt_pkg.config.find_file("Dir::Etc::sourcelist")
+        if os.path.exists(existing):
+            with open(existing, 'r') as fp:
+                for line in fp:
+                    src = SourceEntry(line, sourcelist)
+                    if (src.invalid or src.disabled) or src not in sources:
+                        sources.list.append(src)
+            os.remove(existing)
+
+        for entry in sources.list:
+            if entry.uri.startswith('cdrom:'):
+                # Make sure CD entries come first.
+                sources.list.remove(entry)
+                sources.list.insert(0, entry)
+                entry.disabled = True
         sources.save()
 
-

=== added file 'tests/data/lucid-sources.list'
--- tests/data/lucid-sources.list	1970-01-01 00:00:00 +0000
+++ tests/data/lucid-sources.list	2011-04-06 14:12:27 +0000
@@ -0,0 +1,53 @@
+#deb cdrom:[Ubuntu 10.04.2 LTS _Lucid Lynx_ - Release i386 (20110211.1)]/ lucid main restricted
+# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
+# newer versions of the distribution.
+
+deb http://gb.archive.ubuntu.com/ubuntu/ lucid main restricted
+deb-src http://gb.archive.ubuntu.com/ubuntu/ lucid main restricted
+
+## Major bug fix updates produced after the final release of the
+## distribution.
+deb http://gb.archive.ubuntu.com/ubuntu/ lucid-updates main restricted
+deb-src http://gb.archive.ubuntu.com/ubuntu/ lucid-updates main restricted
+
+## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
+## team. Also, please note that software in universe WILL NOT receive any
+## review or updates from the Ubuntu security team.
+deb http://gb.archive.ubuntu.com/ubuntu/ lucid universe
+deb-src http://gb.archive.ubuntu.com/ubuntu/ lucid universe
+deb http://gb.archive.ubuntu.com/ubuntu/ lucid-updates universe
+deb-src http://gb.archive.ubuntu.com/ubuntu/ lucid-updates universe
+
+## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu 
+## team, and may not be under a free licence. Please satisfy yourself as to 
+## your rights to use the software. Also, please note that software in 
+## multiverse WILL NOT receive any review or updates from the Ubuntu
+## security team.
+deb http://gb.archive.ubuntu.com/ubuntu/ lucid multiverse
+deb-src http://gb.archive.ubuntu.com/ubuntu/ lucid multiverse
+deb http://gb.archive.ubuntu.com/ubuntu/ lucid-updates multiverse
+deb-src http://gb.archive.ubuntu.com/ubuntu/ lucid-updates multiverse
+
+## Uncomment the following two lines to add software from the 'backports'
+## repository.
+## N.B. software from this repository may not have been tested as
+## extensively as that contained in the main release, although it includes
+## newer versions of some applications which may provide useful features.
+## Also, please note that software in backports WILL NOT receive any review
+## or updates from the Ubuntu security team.
+# deb http://gb.archive.ubuntu.com/ubuntu/ lucid-backports main restricted universe multiverse
+# deb-src http://gb.archive.ubuntu.com/ubuntu/ lucid-backports main restricted universe multiverse
+
+## Uncomment the following two lines to add software from Canonical's
+## 'partner' repository.
+## This software is not part of Ubuntu, but is offered by Canonical and the
+## respective vendors as a service to Ubuntu users.
+# deb http://archive.canonical.com/ubuntu lucid partner
+# deb-src http://archive.canonical.com/ubuntu lucid partner
+
+deb http://security.ubuntu.com/ubuntu lucid-security main restricted
+deb-src http://security.ubuntu.com/ubuntu lucid-security main restricted
+deb http://security.ubuntu.com/ubuntu lucid-security universe
+deb-src http://security.ubuntu.com/ubuntu lucid-security universe
+deb http://security.ubuntu.com/ubuntu lucid-security multiverse
+deb-src http://security.ubuntu.com/ubuntu lucid-security multiverse

=== added file 'tests/data/natty-sources.list'
--- tests/data/natty-sources.list	1970-01-01 00:00:00 +0000
+++ tests/data/natty-sources.list	2011-04-06 14:12:27 +0000
@@ -0,0 +1,56 @@
+deb cdrom:[Ubuntu 11.04 _Natty Narwhal_ - Alpha i386 (20110324)]/ natty main restricted
+
+# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
+# newer versions of the distribution.
+deb http://gb.archive.ubuntu.com/ubuntu/ natty main restricted
+deb-src http://gb.archive.ubuntu.com/ubuntu/ natty main restricted
+
+## Major bug fix updates produced after the final release of the
+## distribution.
+deb http://gb.archive.ubuntu.com/ubuntu/ natty-updates main restricted
+deb-src http://gb.archive.ubuntu.com/ubuntu/ natty-updates main restricted
+
+## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
+## team. Also, please note that software in universe WILL NOT receive any
+## review or updates from the Ubuntu security team.
+deb http://gb.archive.ubuntu.com/ubuntu/ natty universe
+deb-src http://gb.archive.ubuntu.com/ubuntu/ natty universe
+deb http://gb.archive.ubuntu.com/ubuntu/ natty-updates universe
+deb-src http://gb.archive.ubuntu.com/ubuntu/ natty-updates universe
+
+## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu 
+## team, and may not be under a free licence. Please satisfy yourself as to 
+## your rights to use the software. Also, please note that software in 
+## multiverse WILL NOT receive any review or updates from the Ubuntu
+## security team.
+deb http://gb.archive.ubuntu.com/ubuntu/ natty multiverse
+deb-src http://gb.archive.ubuntu.com/ubuntu/ natty multiverse
+deb http://gb.archive.ubuntu.com/ubuntu/ natty-updates multiverse
+deb-src http://gb.archive.ubuntu.com/ubuntu/ natty-updates multiverse
+
+## Uncomment the following two lines to add software from the 'backports'
+## repository.
+## N.B. software from this repository may not have been tested as
+## extensively as that contained in the main release, although it includes
+## newer versions of some applications which may provide useful features.
+## Also, please note that software in backports WILL NOT receive any review
+## or updates from the Ubuntu security team.
+# deb http://gb.archive.ubuntu.com/ubuntu/ natty-backports main restricted universe multiverse
+# deb-src http://gb.archive.ubuntu.com/ubuntu/ natty-backports main restricted universe multiverse
+
+deb http://security.ubuntu.com/ubuntu natty-security main restricted
+deb-src http://security.ubuntu.com/ubuntu natty-security main restricted
+deb http://security.ubuntu.com/ubuntu natty-security universe
+deb-src http://security.ubuntu.com/ubuntu natty-security universe
+deb http://security.ubuntu.com/ubuntu natty-security multiverse
+deb-src http://security.ubuntu.com/ubuntu natty-security multiverse
+
+## This software is not part of Ubuntu, but is offered by Canonical and the
+## respective vendors as a service to Ubuntu users.
+deb http://archive.canonical.com/ubuntu natty partner
+deb-src http://archive.canonical.com/ubuntu natty partner
+
+## This software is not part of Ubuntu, but is offered by third-party
+## developers who want to ship their latest software.
+deb http://extras.ubuntu.com/ubuntu natty main
+deb-src http://extras.ubuntu.com/ubuntu natty main

=== added file 'tests/test_merge_sources.py'
--- tests/test_merge_sources.py	1970-01-01 00:00:00 +0000
+++ tests/test_merge_sources.py	2011-04-06 14:12:27 +0000
@@ -0,0 +1,50 @@
+import unittest
+import tempfile
+import os
+import shutil
+import sys
+
+sys.path.insert(0, "..")
+from apt_clone import AptClone
+
+class TestMergeSources(unittest.TestCase):
+    def test_merge_sources(self):
+        clone = AptClone()
+        tmpdir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmpdir)
+        sources_list = os.path.join(tmpdir, "etc", "apt", "sources.list")
+        os.makedirs(os.path.dirname(sources_list))
+        shutil.copy('data/lucid-sources.list', sources_list)
+        backup = os.path.join(tmpdir, "etc", "apt", "sources.list.apt-clone")
+        shutil.copy('data/natty-sources.list', backup)
+        clone._rewrite_sources_list(tmpdir, 'natty')
+        with open(sources_list) as fp:
+            # Tally the occurances of every source line.
+            from collections import defaultdict
+            tally = defaultdict(int)
+            for line in fp:
+                if line != '\n' and not line.startswith('#'):
+                    tally[line] += 1
+            # There should not be any duplicate source lines.
+            for line, count in tally.iteritems():
+                self.failUnless(count == 1, '"%s" occurred %d times.'
+                                % (line, count))
+
+            # Check for extras, others...
+            l = (('partner',
+                  'deb http://archive.canonical.com/ubuntu natty partner\n'),
+                 ('extras',
+                  'deb http://extras.ubuntu.com/ubuntu natty main\n'),
+                 ('main',
+                  'deb http://gb.archive.ubuntu.com/ubuntu/ natty main restricted\n'))
+            for pocket, match in l:
+                fp.seek(0)
+                found = False
+                for line in fp:
+                    if line == match:
+                        found = True
+                self.failUnless(found,
+                        '%s repository not present or disabled.' % pocket)
+
+if __name__ == "__main__":
+    unittest.main()