Merge lp:~jibel/update-manager/AutoUpgradeTester-aptclone into lp:update-manager

[Jean-Baptiste Lallement]

* Added apt-clone support to auto-upgrade tester
* Couple of fixes/tweaks for existing profiles.
* Added profile for lts-main-all-amd64

To test a clone run:
$ ./auto-upgrade-tester clone.tgz
OR
Create a profile with a setting AptClone in the NonInteractive section which is the path to a clone archive.

=== modified file 'AutoUpgradeTester/UpgradeTestBackendQemu.py'
--- AutoUpgradeTester/UpgradeTestBackendQemu.py	2011-12-14 13:48:44 +0000
+++ AutoUpgradeTester/UpgradeTestBackendQemu.py	2012-01-05 08:39:03 +0000
@@ -72,13 +72,14 @@
         if not os.path.exists(self.baseimage):
             print "Missing '%s' base image, need to build it now" % self.baseimage
             arch = self.config.getWithDefault("KVM", "Arch", "i386")
+            rootsize = self.config.getWithDefault("KVM", "RootSize", "80000")
             destdir = "ubuntu-kvm-%s-%s" % (arch, self.fromDist)
             ret = subprocess.call(["sudo",
                                    "ubuntu-vm-builder","kvm", self.fromDist,
                                    "--kernel-flavour", "generic",
                                    "--ssh-key", "%s.pub" % self.ssh_key ,
                                    "--components", "main,restricted",
-                                   "--rootsize", "80000",
+                                   "--rootsize", rootsize,
                                    "--addpkg", "openssh-server",
                                    "--destdir", destdir,
                                    "--arch", arch])
@@ -124,7 +125,7 @@
         self.qemu_options.append("localhost:%s" % str(vncport))
 
         # make the memory configurable
-        mem = self.config.getWithDefault("KVM","VirtualRam","768")
+        mem = self.config.getWithDefault("KVM","VirtualRam","1536")
         self.qemu_options.append("-m")
         self.qemu_options.append(str(mem))
 
@@ -262,18 +263,35 @@
         tzone.flush()
         self._copyToImage(tzone.name, "/etc/timezone")
 
-        # create /etc/apt/sources.list
-        sources = self.getSourcesListFile()
-        self._copyToImage(sources.name, "/etc/apt/sources.list")
-
-        # install some useful stuff
-        ret = self._runInImage(["apt-get","update"])
-        assert ret == 0
-        # FIXME: instead of this retrying (for network errors with 
-        #        proxies) we should have a self._runAptInImage() 
-        for i in range(3):
-            ret = self._runInImage(["DEBIAN_FRONTEND=noninteractive","apt-get","install", "-y",basepkg]+additional_base_pkgs)
-        assert ret == 0
+        aptclone = self.config.getWithDefault('NonInteractive', 'AptCloneFile', '')
+
+        if not aptclone:
+            # create /etc/apt/sources.list
+            sources = self.getSourcesListFile()
+            self._copyToImage(sources.name, "/etc/apt/sources.list")
+
+            # install some useful stuff
+            ret = self._runInImage(["apt-get","update"])
+            assert ret == 0
+            # FIXME: instead of this retrying (for network errors with 
+            #        proxies) we should have a self._runAptInImage() 
+            for i in range(3):
+                ret = self._runInImage(["DEBIAN_FRONTEND=noninteractive","apt-get","install", "-y",basepkg]+additional_base_pkgs)
+            assert ret == 0
+        else:
+            dst_clonename = '/tmp/apt-clone.tgz'
+            self._copyToImage(aptclone, dst_clonename)
+            ret = self._runInImage(["DEBIAN_FRONTEND=noninteractive", "apt-get",
+                                    "install", "-y", "apt-clone"])
+            assert ret == 0
+            print "Restoring clone from %s" % aptclone
+            ret = self._runInImage(['DEBIAN_FRONTEND=noninteractive',
+                                    'apt-clone', 'restore', dst_clonename])
+            # FIXME: what action should be taken when a package failed
+            #        to restore?
+            if ret != 0:
+                print "WARNING: Some packages failed to restore. Continuing anyway!"
+            #assert ret == 0
 
         CMAX = 4000
         pkgs =  self.config.getListFromFile("NonInteractive","AdditionalPkgs")

=== modified file 'AutoUpgradeTester/auto-upgrade-tester'
--- AutoUpgradeTester/auto-upgrade-tester	2011-11-07 12:20:29 +0000
+++ AutoUpgradeTester/auto-upgrade-tester	2012-01-05 08:39:03 +0000
@@ -13,6 +13,8 @@
 import shutil
 import sys
 import time
+import subprocess
+from ConfigParser import SafeConfigParser
 
 # check if we run from a bzr checkout
 if os.path.exists("./UpgradeTestBackend.py"):
@@ -209,6 +211,52 @@
     os.dup2(old_stderr, 2)
     return result
 
+def profileFromAptClone(aptclonefile):
+    """ Create a profile from an apt-clone archive
+
+    :param aptclonefile: Path to an apt-clone archive
+    :return: path to profile or None if it fails to recognize the clone file
+    """
+    profile = None
+    try:
+        cmd = ('apt-clone', 'info', aptclonefile)
+        output = subprocess.check_output(cmd)
+        clone_properties = {}
+        for k, v in [ line.split(': ') for line in output.split('\n') if ': ' in line]:
+            clone_properties[k] = v
+        try:
+            clone_date = datetime.datetime.strptime(clone_properties['Date'], '%c').strftime('_%Y%m%d-%H%M%S')
+        except:
+            clone_date = ''
+        profilename = "%s_%s_%s%s" % (
+            clone_properties['Distro'],
+            clone_properties['Arch'],
+            clone_properties['Hostname'],
+            clone_date)
+
+        # Generate configuration file for this profile
+        # FIXME:
+        # update packaging to grant write access on base for upgrade-tester user
+        ppath = 'profile' if os.path.exists("./UpgradeTestBackend.py") else base
+        profile = os.path.join(ppath, profilename)
+        profilecfg = SafeConfigParser()
+        if not os.path.exists(profile):
+            os.makedirs(profile)
+
+        profilecfg.set('DEFAULT', 'SourceRelease', clone_properties['Distro'])
+        profilecfg.add_section('NonInteractive')
+        profilecfg.set('NonInteractive', 'ProfileName', profilename)
+        profilecfg.set('NonInteractive', 'AptcloneFile', os.path.abspath(aptclonefile))
+        profilecfg.add_section('KVM')
+        profilecfg.set('KVM', 'Arch', clone_properties['Arch'])
+        with open(os.path.join(profile, 'DistUpgrade.cfg'), 'wb') as profilefile:
+            profilecfg.write(profilefile)
+
+    except subprocess.CalledProcessError:
+        return None
+
+    return profile
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
 
@@ -279,6 +327,16 @@
     html_output = HtmlOutputStream(options.html_output_dir)
     html_output.write_html_header(time_started)
     for profile in profiles:
+
+        # case of an apt-clone archive
+        if os.path.isfile(profile):
+            profilename = profileFromAptClone(profile)
+            if not profilename:
+                print "File is not a valid apt-clone image: %s" % profilename
+                continue
+            profile = profilename
+            print "Using generated profile: %s" % profile
+
         if not "/" in profile:
             profile = base + profile
         try:

=== modified file 'AutoUpgradeTester/post_upgrade_tests/python_import_test.py'
--- AutoUpgradeTester/post_upgrade_tests/python_import_test.py	2011-11-08 08:23:45 +0000
+++ AutoUpgradeTester/post_upgrade_tests/python_import_test.py	2012-01-05 08:39:03 +0000
@@ -29,6 +29,7 @@
              "keyring",
              "invest",
              "Onboard",
+             "goocanvasmodule.so",
              ]
 
 def get_module_from_path(path):
@@ -82,10 +83,13 @@
 if __name__ == "__main__":
     #logging.basicConfig(level=logging.DEBUG)
 
-    old_modules = set(filter(py_module_filter, os.listdir(OLD_BASEPATH)))
-    new_modules = set(filter(py_module_filter, os.listdir(NEW_BASEPATH)))
-    print "Available for the old version, but *not* the new: %s\n" % (
-        ", ".join(old_modules - new_modules))
+    # Only compare if old and new paths exists
+    # When previous version of python is dropped then only new exists
+    if os.path.exists(OLD_BASEPATH) and os.path.exists(NEW_BASEPATH):
+        old_modules = set(filter(py_module_filter, os.listdir(OLD_BASEPATH)))
+        new_modules = set(filter(py_module_filter, os.listdir(NEW_BASEPATH)))
+        print "Available for the old version, but *not* the new: %s\n" % (
+            ", ".join(old_modules - new_modules))
 
     res = True
     # FIXME: instead os os.listdir() use os.walk() to catch subdirs

=== modified file 'AutoUpgradeTester/profile/defaults.cfg.d/defaults.cfg'
--- AutoUpgradeTester/profile/defaults.cfg.d/defaults.cfg	2011-11-07 14:29:56 +0000
+++ AutoUpgradeTester/profile/defaults.cfg.d/defaults.cfg	2012-01-05 08:39:03 +0000
@@ -40,6 +40,8 @@
 CacheBaseImage=yes
 VncNum=1
 SshPort=54322
+VirtualRam=1536
+RootSize=80000
 
 [EC2]
 ; Ubuntu official images:

=== added directory 'AutoUpgradeTester/profile/lts-main-all-amd64'
=== added file 'AutoUpgradeTester/profile/lts-main-all-amd64/DistUpgrade.cfg'
--- AutoUpgradeTester/profile/lts-main-all-amd64/DistUpgrade.cfg	1970-01-01 00:00:00 +0000
+++ AutoUpgradeTester/profile/lts-main-all-amd64/DistUpgrade.cfg	2012-01-05 08:39:03 +0000
@@ -0,0 +1,12 @@
+[DEFAULT]
+SourceRelease=lucid
+
+[NonInteractive]
+ProfileName = lts-main-all-amd64
+AdditionalPkgs = pkgs.cfg
+PostBootstrapScript=/usr/share/pyshared/AutoUpgradeTester/install_all.py
+;PostBootstrapScript=/home/upgrade-tester/update-manager/AutoUpgradeTester/install_all.py
+
+[KVM]
+Arch=amd64
+

=== added file 'AutoUpgradeTester/profile/lts-main-all-amd64/DistUpgrade.cfg.dapper'
--- AutoUpgradeTester/profile/lts-main-all-amd64/DistUpgrade.cfg.dapper	1970-01-01 00:00:00 +0000
+++ AutoUpgradeTester/profile/lts-main-all-amd64/DistUpgrade.cfg.dapper	2012-01-05 08:39:03 +0000
@@ -0,0 +1,65 @@
+[View]
+#View=DistUpgradeViewGtk
+View=DistUpgradeViewNonInteractive
+
+# Distro contains global information about the upgrade
+[Distro]
+# the meta-pkgs we support
+MetaPkgs=ubuntu-desktop, kubuntu-desktop, edubuntu-desktop, xubuntu-desktop
+BaseMetaPkgs=ubuntu-minimal, ubuntu-standard
+PostUpgradePurge=xorg-common, libgl1-mesa
+Demotions=demoted.cfg
+RemoveEssentialOk=sysvinit
+RemovalBlacklistFile=removal_blacklist.cfg
+
+# information about the individual meta-pkgs
+[ubuntu-desktop]
+KeyDependencies=gdm, gnome-panel, ubuntu-artwork
+# those pkgs will be marked remove right after the distUpgrade in the cache
+PostUpgradeRemove=xchat, xscreensaver
+
+[kubuntu-desktop]
+KeyDependencies=kdm, kicker, kubuntu-artwork-usplash
+# those packages are marked as obsolete right after the upgrade
+ForcedObsoletes=ivman
+
+[edubuntu-desktop]
+KeyDependencies=edubuntu-artwork, tuxpaint
+
+[xubuntu-desktop]
+KeyDependencies=xubuntu-artwork-usplash, xubuntu-default-settings, xfce4
+
+
+[Files]
+BackupExt=distUpgrade
+
+[Sources]
+From=dapper
+To=hardy
+ValidOrigin=Ubuntu
+ValidMirrors = mirrors.cfg
+
+[Network]
+MaxRetries=3
+
+[PreRequists]
+Packages=release-upgrader-apt,release-upgrader-dpkg
+SourcesList=prerequists-sources.list
+
+[NonInteractive]
+ProfileName = main-all
+BasePkg = ubuntu-standard
+AdditionalPkgs = pkgs.cfg
+Mirror = http://archive.ubuntu.com/ubuntu
+Proxy=http://192.168.1.1:3128/
+ForceOverwrite=yes
+CacheTarball=yes
+PostBootstrapScript=install_all.py
+Components=main,restricted
+Pockets=security,updates
+BaseImage=jeos/dapper-i386.qcow2
+CacheBaseImage=yes
+SwapImage=jeos/swap.qcow2
+UpgradeFromDistOnBootstrap=true
+SSHKey=ssh-key
+ReadReboot=yes
\ No newline at end of file

=== added file 'AutoUpgradeTester/profile/lts-main-all-amd64/demoted.cfg'
=== added file 'AutoUpgradeTester/profile/lts-main-all-amd64/install_blacklist.cfg'
--- AutoUpgradeTester/profile/lts-main-all-amd64/install_blacklist.cfg	1970-01-01 00:00:00 +0000
+++ AutoUpgradeTester/profile/lts-main-all-amd64/install_blacklist.cfg	2012-01-05 08:39:03 +0000
@@ -0,0 +1,46 @@
+# jaunty: endless loop with DEBIAN_FRONTEND=noninteractive
+moodle
+# has a funny "can not be upgraded automatically" policy
+# see debian #368226
+quagga
+# not installable on a regular machine (preinst error)
+ltsp-client
+# gwenview-i18n has some file conflicts
+gwenview-i18n
+# ipppd needs MAKEDEV
+ipppd
+# cmake has incorrect emacs dependencies
+cmake
+# cluster manager hangs on shutdown
+cman
+
+# Conflicts/uninstallable packages in Oneiric
+libgladeui-doc
+mythes-it
+amavisd-new
+grub-legacy-ec2
+
+# LP:#901638
+tdsodbc
+
+# Only useful on livecd
+casper
+ubiquity
+
+# No need for so many kernels
+linux-image-.*
+
+# Failed to install
+bacula-director-mysql
+dovecot-common
+rabbitmq-server
+
+# cloud/ec2 no fun
+cloud-init
+ec2-init
+linux-image-ec2
+linux-image-2.6.31-302-ec2
+
+# not installable on a regular machine
+ltsp-client
+ltsp-client-core

=== added file 'AutoUpgradeTester/profile/lts-main-all-amd64/pkgs.cfg'
--- AutoUpgradeTester/profile/lts-main-all-amd64/pkgs.cfg	1970-01-01 00:00:00 +0000
+++ AutoUpgradeTester/profile/lts-main-all-amd64/pkgs.cfg	2012-01-05 08:39:03 +0000
@@ -0,0 +1,3 @@
+python-apt
+python-gnupginterface
+grub-pc

=== added file 'AutoUpgradeTester/profile/lts-main-all-amd64/prerequists-sources.list'
--- AutoUpgradeTester/profile/lts-main-all-amd64/prerequists-sources.list	1970-01-01 00:00:00 +0000
+++ AutoUpgradeTester/profile/lts-main-all-amd64/prerequists-sources.list	2012-01-05 08:39:03 +0000
@@ -0,0 +1,5 @@
+# sources.list fragment for pre-requists (one with countrymirror, one fallback)
+deb http://archive.ubuntu.com/ubuntu dapper-backports main/debian-installer
+#deb http://archive.ubuntu.com/ubuntu feisty-backports main/debian-installer
+# below is just for testing
+#deb http://archive.dogfood.launchpad.net/ubuntu feisty-backports main/debian-installer

=== added file 'AutoUpgradeTester/profile/lts-main-all-amd64/removal_blacklist.cfg'
--- AutoUpgradeTester/profile/lts-main-all-amd64/removal_blacklist.cfg	1970-01-01 00:00:00 +0000
+++ AutoUpgradeTester/profile/lts-main-all-amd64/removal_blacklist.cfg	2012-01-05 08:39:03 +0000
@@ -0,0 +1,11 @@
+# blacklist of packages that should never be removed
+ubuntu-standard
+ubuntu-minimal
+ubuntu-desktop
+kubuntu-desktop
+edubuntu-desktop
+gobuntu-desktop
+# never remove nvidia-glx and friends
+^nvidia-glx$
+^nvidia-glx-new$
+^nvidia-glx-legacy$

=== modified file 'AutoUpgradeTester/profile/lts-main-all/DistUpgrade.cfg'
--- AutoUpgradeTester/profile/lts-main-all/DistUpgrade.cfg	2011-12-11 22:39:17 +0000
+++ AutoUpgradeTester/profile/lts-main-all/DistUpgrade.cfg	2012-01-05 08:39:03 +0000
@@ -5,4 +5,4 @@
 ProfileName = lts-main-all
 AdditionalPkgs = pkgs.cfg
 PostBootstrapScript=/usr/share/pyshared/AutoUpgradeTester/install_all.py
-
+;PostBootstrapScript=/home/upgrade-tester/update-manager/AutoUpgradeTester/install_all.py

=== added file 'AutoUpgradeTester/profile/lts-main-all/install_blacklist.cfg'
--- AutoUpgradeTester/profile/lts-main-all/install_blacklist.cfg	1970-01-01 00:00:00 +0000
+++ AutoUpgradeTester/profile/lts-main-all/install_blacklist.cfg	2012-01-05 08:39:03 +0000
@@ -0,0 +1,46 @@
+# jaunty: endless loop with DEBIAN_FRONTEND=noninteractive
+moodle
+# has a funny "can not be upgraded automatically" policy
+# see debian #368226
+quagga
+# not installable on a regular machine (preinst error)
+ltsp-client
+# gwenview-i18n has some file conflicts
+gwenview-i18n
+# ipppd needs MAKEDEV
+ipppd
+# cmake has incorrect emacs dependencies
+cmake
+# cluster manager hangs on shutdown
+cman
+
+# Conflicts/uninstallable packages in Oneiric
+libgladeui-doc
+mythes-it
+amavisd-new
+grub-legacy-ec2
+
+# LP:#901638
+tdsodbc
+
+# Only useful on livecd
+casper
+ubiquity
+
+# No need for so many kernels
+linux-image-.*
+
+# Failed to install
+bacula-director-mysql
+dovecot-common
+rabbitmq-server
+
+# cloud/ec2 no fun
+cloud-init
+ec2-init
+linux-image-ec2
+linux-image-2.6.31-302-ec2
+
+# not installable on a regular machine
+ltsp-client
+ltsp-client-core

=== modified file 'AutoUpgradeTester/profile/lts-main-all/pkgs.cfg'
--- AutoUpgradeTester/profile/lts-main-all/pkgs.cfg	2007-12-17 10:44:16 +0000
+++ AutoUpgradeTester/profile/lts-main-all/pkgs.cfg	2012-01-05 08:39:03 +0000
@@ -1,2 +1,3 @@
 python-apt
 python-gnupginterface
+grub-pc

=== modified file 'AutoUpgradeTester/profile/lts-ubuntu/pkgs.cfg'
--- AutoUpgradeTester/profile/lts-ubuntu/pkgs.cfg	2008-02-01 14:13:41 +0000
+++ AutoUpgradeTester/profile/lts-ubuntu/pkgs.cfg	2012-01-05 08:39:03 +0000
@@ -1,2 +1,3 @@
 ubuntu-minimal
-ubuntu-standard
\ No newline at end of file
+ubuntu-standard
+grub-pc

=== modified file 'AutoUpgradeTester/profile/main-all-amd64/DistUpgrade.cfg'
--- AutoUpgradeTester/profile/main-all-amd64/DistUpgrade.cfg	2011-12-11 22:39:17 +0000
+++ AutoUpgradeTester/profile/main-all-amd64/DistUpgrade.cfg	2012-01-05 08:39:03 +0000
@@ -2,6 +2,7 @@
 ProfileName = main-all-amd64
 AdditionalPkgs = pkgs.cfg
 PostBootstrapScript=/usr/share/pyshared/AutoUpgradeTester/install_all.py
+;PostBootstrapScript=/home/upgrade-tester/update-manager/AutoUpgradeTester/install_all.py
 
 [KVM]
 Arch=amd64

=== modified file 'AutoUpgradeTester/profile/main-all-amd64/install_blacklist.cfg'
--- AutoUpgradeTester/profile/main-all-amd64/install_blacklist.cfg	2011-12-03 23:30:58 +0000
+++ AutoUpgradeTester/profile/main-all-amd64/install_blacklist.cfg	2012-01-05 08:39:03 +0000
@@ -19,3 +19,11 @@
 mythes-it
 amavisd-new
 grub-legacy-ec2
+
+# LP:#901638
+tdsodbc
+
+# Only useful on livecd
+casper
+ubiquity
+

=== modified file 'AutoUpgradeTester/profile/main-all/DistUpgrade.cfg'
--- AutoUpgradeTester/profile/main-all/DistUpgrade.cfg	2011-12-11 22:39:17 +0000
+++ AutoUpgradeTester/profile/main-all/DistUpgrade.cfg	2012-01-05 08:39:03 +0000
@@ -2,3 +2,4 @@
 ProfileName = main-all
 AdditionalPkgs = pkgs.cfg
 PostBootstrapScript=/usr/share/pyshared/AutoUpgradeTester/install_all.py
+;PostBootstrapScript=/home/upgrade-tester/update-manager/AutoUpgradeTester/install_all.py

=== modified file 'AutoUpgradeTester/profile/main-all/install_blacklist.cfg'
--- AutoUpgradeTester/profile/main-all/install_blacklist.cfg	2011-12-03 23:30:58 +0000
+++ AutoUpgradeTester/profile/main-all/install_blacklist.cfg	2012-01-05 08:39:03 +0000
@@ -19,3 +19,10 @@
 mythes-it
 amavisd-new
 grub-legacy-ec2
+
+# LP:#901638
+tdsodbc
+
+# Only useful on livecd
+casper
+ubiquity