Merge lp:~nuclearbob/utah/iso-download into lp:utah

[Max Brustkern]

This branch adds support for automatically downloading an image by passing series, installtype, and arch to the ISO class. It also changes CustomVM to create a temporary copy of the image it uses to install so multiple simultaneous runs don't get hung up with libvirt trying to lock the image. It also sets CustomVM as the default VM provisioning method for all cases, putting us in a position to deprecate VMToolsVM since vm-tools has been replaced.

[lp:~nuclearbob/utah/iso-download]

705. By Max Brustkern on 2012-11-08

Added comments to new config options

[Joe Talbott (joetalbott)]

Looks good to me.

review: Approve

=== modified file 'debian/control'
--- debian/control	2012-10-25 20:07:09 +0000
+++ debian/control	2012-11-08 20:12:18 +0000
@@ -17,7 +17,7 @@
          python-apt, python-libvirt,
          python-netifaces, python-paramiko, python-psutil,
          utah-client (=${binary:Version})
-Recommends: kvm
+Recommends: dl-ubuntu-test-iso, kvm
 Suggests: cobbler, u-boot-tools, vm-tools
 Description: Ubuntu Test Automation Harness
  Automation framework for testing in Ubuntu

=== modified file 'examples/run_utah_tests.py'
--- examples/run_utah_tests.py	2012-10-19 21:57:01 +0000
+++ examples/run_utah_tests.py	2012-11-08 20:12:18 +0000
@@ -7,7 +7,7 @@
 from utah.group import check_user_group, print_group_error_message
 from utah.timeout import timeout, UTAHTimeout
 from utah import config
-from run_test_vm import run_test_vm
+from run_install_test import run_install_test
 
 
 def get_parser():
@@ -87,21 +87,16 @@
     if args is None:
         args = get_parser().parse_args()
 
+    # Default is now CustomVM
+    function = run_install_test
     if args.arch is not None and 'arm' in args.arch:
+        # If arch is arm, use BambooFeederMachine
         from run_test_bamboo_feeder import run_test_bamboo_feeder
         function = run_test_bamboo_feeder
     elif args.machinetype == 'physical':
+        # If machinetype is physical but arch isn't arm, use CobblerMachine
         from run_test_cobbler import run_test_cobbler
         function = run_test_cobbler
-    else:
-        function = run_test_vm
-
-        for option in [args.boot, args.emulator, args.image, args.initrd,
-                       args.kernel, args.preseed, args.xml]:
-            if option is not None:
-                from run_install_test import run_install_test
-                function = run_install_test
-                break
     
     function(args=args)
 

=== modified file 'utah/config.py'
--- utah/config.py	2012-11-08 16:27:38 +0000
+++ utah/config.py	2012-11-08 20:12:18 +0000
@@ -31,6 +31,10 @@
     checktimeout=15,
     consoleloglevel=logging.WARNING,
     debug=False,
+    # Command to download test images
+    dlcommand='dl-ubuntu-test-iso',
+    # Number of times to retry image download
+    dlretries=10,
     fileloglevel=logging.INFO,
     group='utah',
     image=None,
@@ -38,6 +42,8 @@
     installpackages=['openssh-server', 'gdebi'],
     installtimeout=3600,
     installtype='desktop',
+    # Directory to store local images
+    isodir='/var/cache/utah/iso',
     kernel=None,
     logpath=os.path.join('/', 'var', 'log', 'utah'),
     machineid=None,

=== modified file 'utah/iso.py'
--- utah/iso.py	2012-08-30 08:45:59 +0000
+++ utah/iso.py	2012-11-08 20:12:18 +0000
@@ -1,20 +1,24 @@
 """
 All commands use bsdtar and accept a logmethod to use for logging.
 """
-#TODO: Make an iso class that can return things like arch, series, type, etc.
-#TODO: Add the existing functions to the iso class.
-#TODO: Convert the code to use the class instead of the existing functions.
 import logging
 import logging.handlers
 import os
+import shutil
 import subprocess
 import sys
 import urllib
 
+from hashlib import md5
+
 from utah import config
 from utah.exceptions import UTAHException
 
 
+# TODO: eliminate usage of the non-class functions
+# TODO: move the non-class functions into the class
+
+
 def listfiles(image, logmethod=None, returnlist=False):
     """
     Return either a subprocess instance listing the contents of an iso, or a
@@ -115,18 +119,28 @@
     """
     Provide a simplified method of interfacing with images.
     """
-    def __init__(self, image, dlpercentincrement=1, logger=None):
+    def __init__(self, arch=None, dlpercentincrement=1, dlretries=None,
+                 image=None, installtype=None, logger=None, series=None):
         self.dlpercentincrement = dlpercentincrement
+        if dlretries is None:
+            self.dlretries = config.dlretries
+        else:
+            self.dlretries = dlretries
         if logger is None:
             self._loggersetup()
         else:
             self.logger = logger
-        if image.startswith('~'):
-            path = os.path.expanduser(image)
-            self.logger.debug('Rewriting ~-based path: ' + image +
-                              ' to: ' + path)
+        if image is None:
+            self.logger.debug('Need to download image')
+            path = self.downloadiso(arch=arch, installtype=installtype,
+                                    series=series)
         else:
-            path = image
+            if image.startswith('~'):
+                path = os.path.expanduser(image)
+                self.logger.debug('Rewriting ~-based path: ' + image +
+                                  ' to: ' + path)
+            else:
+                path = image
 
         self.percent = 0
         self.logger.info('Preparing image: ' + path)
@@ -218,6 +232,9 @@
         return series
 
     def dldisplay(self, blocks, size, total):
+        """
+        Log download information.
+        """
         read = blocks * size
         percent = 100 * read / total
         if percent >= self.percent:
@@ -226,19 +243,140 @@
         self.logger.debug("%s read, %s%% of %s total" % (read, percent, total))
 
     def listfiles(self, returnlist=False):
+        """
+        List all files in the image.
+        """
         return listfiles(self.image, self.logger.debug, returnlist=returnlist)
 
     def getrealfile(self, filename, outdir=None):
+        """
+        Return the real path to a file (following links if needed.)
+        """
         return getrealfile(self.image, filename, outdir=outdir,
                            logmethod=self.logger.debug)
 
     def extract(self, filename, outdir=None, outfile=None, **kw):
+        """
+        Extract a file.
+        """
         return extract(self.image, filename, outdir, outfile,
                        logmethod=self.logger.debug, **kw)
 
     def dump(self, filename, **kw):
+        """
+        Dump a file.
+        """
         return dump(self.image, filename, logmethod=self.logger.debug, **kw)
 
     def extractall(self):
+        """
+        Extract all files in the image.
+        """
         for myfile in self.listfiles(returnlist=True):
             self.extract(myfile)
+            
+    def getmd5(self, path):
+        """
+        Return the md5 checksum of a file.
+        """
+        if path is None:
+            path = self.image
+        self.logger.debug('Getting md5 of ' + path)
+        isohash = md5()
+        with open(path) as myfile:
+            for block in iter(lambda: myfile.read(128), ""):
+                isohash.update(block)
+        filemd5 = isohash.hexdigest()
+        self.logger.debug('md5 of ' + path + ' is ' + filemd5)
+        return filemd5
+        
+    def downloadiso(self, arch=None, installtype=None, series=None):
+        """
+        Download an ISO given series, type, and arch.
+        """
+        if arch is None:
+            arch = config.arch
+        if installtype is None:
+            installtype = config.installtype
+        if series is None:
+            series = config.series
+        filename = '-'.join([series, installtype, arch]) + '.iso'
+        self.logger.info('Attempting to download ' + filename)
+        with open(os.devnull, "w") as fnull:
+            # If dlcommand (default dl-ubuntu-test-iso) is available, use it
+            if subprocess.call(['which', config.dlcommand],
+                               stdout=fnull, stderr=fnull) == 0:
+                self.logger.debug('Using ' + config.dlcommand)
+                if installtype == 'server':
+                    flavor = 'ubuntu-server'
+                else:
+                    flavor = 'ubuntu'
+                cmd = [config.dlcommand,
+                       '-q',
+                       '--flavor=' + flavor,
+                       '--release=' + series,
+                       '--variant=' + installtype,
+                       '--arch=' + arch,
+                       '--isoroot=' + config.isodir]
+                self.logger.info('Downloading ISO')
+                self.logger.debug(' '.join(cmd))
+                subprocess.check_call(cmd)
+                path = os.path.join(config.isodir, flavor, filename)
+                if os.path.isfile(path):
+                    return path
+
+        # If we haven't returned, dlcommand didn't give us an image
+        # We'll need to use the utah image cache in config.isodir
+        path = os.path.join(config.isodir, filename)
+        # We want to verify our image matches the latest md5 from the server
+        # To do this, we'll loop
+        # First, we'll check the image.  If it matches, we return
+        # If not, we'll download it, and start the loop over
+        for attempt in range(config.dlretries):
+            self.logger.info('Download attempt ' + str(attempt))
+            # Set the path to our files and the name of the iso we want
+            if installtype == 'mini':
+                remotepath = ('http://archive.ubuntu.com/ubuntu/dists/'
+                        '{series}/main/installer-{arch}/current/'
+                        'images/'.format(arch=arch, series=series))
+                isopattern = 'netboot/mini.iso'
+            else:
+                remotepath = ('http://releases.ubuntu.com/'
+                              '{series}/'.format(series=series))
+                isopattern = installtype + '-' + arch + '.iso'
+
+            # Scan the MD5SUMS file for our file
+            servermd5 = None
+            for line in urllib.urlopen(remotepath + '/MD5SUMS'):
+                if isopattern in line:
+                    servermd5, isofile = line.split()
+                    isofile = isofile.strip('*')
+                    break
+
+            # If we can't find the ISO, raise an exception
+            if servermd5 is None:
+                raise UTAHException('Specified ISO: ' + filename
+                                    + ' not found on mirrors.')
+            self.logger.debug('Server md5 is ' + servermd5)
+
+            # If the ISO exists locally and the md5 matches, use it
+            if os.path.isfile(path) and servermd5 == self.getmd5(path):
+                self.logger.info('Using ISO at ' + path)
+                return path
+
+            # Try to download the ISO
+            isopath = os.path.join(remotepath, isofile)
+            self.percent = 0
+            self.logger.info('Attempting to download ' + isopath)
+            temppath = urllib.urlretrieve(isopath, reporthook=self.dldisplay)[0]
+
+            # Try to copy it into our cache
+            self.logger.debug('Copying ' + temppath + ' to ' + path)
+            shutil.copyfile(temppath, path)
+        else:
+            if os.path.isfile(path) and servermd5 == self.getmd5(path):
+                return path
+            else:
+                raise UTAHException('Image failed to download after ' +
+                                    config.dlretries + ' tries')
+            

=== modified file 'utah/provisioning/inventory/sqlite.py'
--- utah/provisioning/inventory/sqlite.py	2012-10-19 13:04:40 +0000
+++ utah/provisioning/inventory/sqlite.py	2012-11-08 20:12:18 +0000
@@ -5,7 +5,7 @@
 from utah.provisioning.inventory.exceptions import \
         UTAHProvisioningInventoryException
 from utah.provisioning.inventory.inventory import Inventory
-from utah.provisioning.vm.libvirtvm import VMToolsVM
+from utah.provisioning.vm.libvirtvm import CustomVM
 from utah.provisioning.baremetal.cobbler import CobblerMachine
 
 
@@ -42,7 +42,7 @@
                 'CREATE TABLE IF NOT EXISTS '
                 'machines(machineid INTEGER PRIMARY KEY, state TEXT)')
 
-    def request(self, machinetype=VMToolsVM, *args, **kw):
+    def request(self, machinetype=CustomVM, *args, **kw):
         """
         Takes a Machine class as machinetype, and passes the newly generated
         machineid along with all other arguments to that class's constructor,

=== modified file 'utah/provisioning/provisioning.py'
--- utah/provisioning/provisioning.py	2012-10-25 20:07:44 +0000
+++ utah/provisioning/provisioning.py	2012-11-08 20:12:18 +0000
@@ -137,12 +137,11 @@
 
         fileargs = ['preseed', 'xml', 'kernel', 'initrd']
 
-        if image is None:
-            self.image = None
-        elif image.endswith('.iso'):
-            self.image = ISO(image,
+        if image is None or image.endswith('.iso'):
+            self.image = ISO(arch=arch,
                     dlpercentincrement=self.dlpercentincrement,
-                    logger=self.logger)
+                    image=image, installtype=installtype, logger=self.logger,
+                    series=series)
         else:
             fileargs.append('image')
 

=== modified file 'utah/provisioning/vm/libvirtvm.py'
--- utah/provisioning/vm/libvirtvm.py	2012-10-19 21:57:01 +0000
+++ utah/provisioning/vm/libvirtvm.py	2012-11-08 20:12:18 +0000
@@ -454,7 +454,7 @@
                         return True
         return False
 
-    def _installxml(self, cmdline=None, initrd=None,
+    def _installxml(self, cmdline=None, image=None, initrd=None,
                     kernel=None, tmpdir=None, xml=None):
         """
         Return the XML tree to be passed to libvirt for VM installation.
@@ -462,6 +462,8 @@
         self.logger.info('Creating installation XML')
         if cmdline is None:
             cmdline = self.cmdline
+        if image is None:
+            image = self.image.image
         if initrd is None:
             initrd = self.initrd
         if kernel is None:
@@ -475,7 +477,8 @@
             self.logger.debug('Setting VM to shutdown on reboot')
             xmlt.find('on_reboot').text = 'destroy'
             if self.rewrite == 'all':
-                self._installxml_rewrite_all(cmdline, initrd, kernel, xmlt)
+                self._installxml_rewrite_all(cmdline, image, initrd, kernel,
+                                             xmlt)
         else:
             self.logger.info('Not rewriting XML because rewrite is ' +
                              self.rewrite)
@@ -484,8 +487,8 @@
         self.logger.info('Installation XML ready')
         return xmlt
 
-    def _installxml_rewrite_all(self, cmdline_txt, initrd_txt, kernel_txt,
-                                xmlt):
+    def _installxml_rewrite_all(self, cmdline_txt, image, initrd_txt,
+                                kernel_txt, xmlt):
         """
         Rewrite the whole configuration file for the VM
         """
@@ -528,11 +531,11 @@
             #TODO: Add a cdrom if none exists
             if disk.get('device') == 'cdrom':
                 if disk.find('source') is not None:
-                    disk.find('source').set('file', self.image.image)
+                    disk.find('source').set('file', image)
                     self.logger.debug('Rewrote existing CD-ROM')
                 else:
                     source = ElementTree.Element('source')
-                    source.set('file', self.image.image)
+                    source.set('file', image)
                     disk.append(source)
                     self.logger.debug('Added source to existing '
                                       'CD-ROM')
@@ -631,6 +634,9 @@
                     ose.remove(cmdline)
                 devices = xml.find('devices')
                 devices.remove(devices.find('serial'))
+                for disk in list(devices.iterfind('disk')):
+                    if disk.get('device') == 'cdrom':
+                        disk.remove(disk.find('source'))
         else:
             self.logger.info('Not rewriting XML because rewrite is ' +
                              self.rewrite)
@@ -638,6 +644,21 @@
             xml.write(os.path.join(tmpdir, 'final.xml'))
         return xml
 
+    def _tmpimage(self, image=None, tmpdir=None):
+        """
+        Create a temporary copy of the image so libvirt will lock that copy.
+        This allows other simultaneous processes to update the cached image.
+        """
+        if image is None:
+            image = self.image.image
+        if tmpdir is None:
+            tmpdir = self.tmpdir
+        self.logger.info('Making temp copy of install image')
+        tmpimage = os.path.join(tmpdir, os.path.basename(image))
+        self.logger.debug('Copying ' + image + ' to ' + tmpimage)
+        shutil.copyfile(image, tmpimage)
+        return tmpimage
+
     def _create(self):
         """
         Create the VM, install the system, and prepare it to boot.
@@ -669,8 +690,10 @@
 
         self._createdisks()
 
-        xml = self._installxml(cmdline=self.cmdline, initrd=initrd,
-                               kernel=kernel, tmpdir=tmpdir, xml=self.xml)
+        image = self._tmpimage(image=self.image.image, tmpdir=tmpdir)
+
+        xml = self._installxml(cmdline=self.cmdline, image=image,
+                initrd=initrd, kernel=kernel, tmpdir=tmpdir, xml=self.xml)
 
         self._installvm(lv=self.lv, tmpdir=tmpdir, xml=xml)