Merge lp:~nuclearbob/utah/iso-static-validation-updates into lp:utah

[Max Brustkern]

This branch makes several updates to the ISO static validation code. It updated it to use the new ISO class. It allows it to run from an arbitrary directory. It allows it to run on ISOs not named series-type-arch.iso. It also updated iso.py to move all the functions into the ISO class since they are now not used externally by anything.

[lp:~nuclearbob/utah/iso-static-validation-updates]

739. By Max Brustkern on 2012-11-13

Updated config handling: directories now supported, as well as most
command line options

[Joe Talbott (joetalbott)]

Looks good to me.

review: Approve

[lp:~nuclearbob/utah/iso-static-validation-updates]

740. By Max Brustkern on 2012-11-14

Added config import to example scripts sop serieschoices can be pulled in. The failure to generate options was breaking the build, so I'm comitting this directly.

741. By Max Brustkern on 2012-11-14

Added support for automatic ISO downloading and made CustomVM the
default provisioning method

742. By Max Brustkern on 2012-11-14

Added support for desktop image kernel detection from bootloader
configuration files

743. By Max Brustkern on 2012-11-14

Updated ISO static validation to use new features of ISO class and
work independent of script directory and image directory and image
name

=== modified file 'debian/utah.install'
--- debian/utah.install	2012-10-05 14:09:09 +0000
+++ debian/utah.install	2012-11-14 16:06:21 +0000
@@ -1,5 +1,6 @@
 conf/config var/lib/utah/.ssh
 conf/utah etc
 examples usr/share/utah
+utah/isotest usr/share/utah
 utah-client_*_all.deb usr/share/utah
 python-jsonschema_*_all.deb usr/share/utah

=== modified file 'utah/iso.py'
--- utah/iso.py	2012-08-30 08:45:59 +0000
+++ utah/iso.py	2012-11-14 16:06:21 +0000
@@ -1,9 +1,6 @@
 """
 All commands use bsdtar and accept a logmethod to use for logging.
 """
-#TODO: Make an iso class that can return things like arch, series, type, etc.
-#TODO: Add the existing functions to the iso class.
-#TODO: Convert the code to use the class instead of the existing functions.
 import logging
 import logging.handlers
 import os
@@ -15,102 +12,6 @@
 from utah.exceptions import UTAHException
 
 
-def listfiles(image, logmethod=None, returnlist=False):
-    """
-    Return either a subprocess instance listing the contents of an iso, or a
-    list of files in the iso if returnlist is True.
-    """
-    cmd = ['bsdtar', '-t', '-f', image]
-    if logmethod is not None:
-        logmethod('bsdtar list command: ' + ' '.join(cmd))
-    if returnlist:
-        return subprocess.check_output(cmd).strip().split('\n')
-    else:
-        return subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE).communicate()
-
-
-def getrealfile(image, filename, outdir=None, logmethod=None):
-    """
-    Return a command to safely extract a file from an iso.
-    Based on unbsdtar-safelink from ubuntu iso testing.
-    """
-    cmd = ['bsdtar', '-t', '-v', '-f', image, filename]
-    if logmethod is not None:
-        logmethod('bsdtar list command: ' + ' '.join(cmd))
-    output = (subprocess.Popen(cmd, stdout=subprocess.PIPE)
-              .communicate()[0].split('\n')[0].split())
-    try:
-        if output[0].startswith('l'):
-            if outdir is None:
-                cmd = ['ln', '-s', output[-1], output[-3]]
-            else:
-                cmd = ['ln', '-s', output[-1],
-                       os.path.join(outdir, output[-3])]
-        else:
-            realfile = output[-1]
-            if outdir is None:
-                cmd = ['bsdtar', '-x', '-f', image, '-O', realfile]
-            else:
-                cmd = ['bsdtar', '-x', '-f', image, '-C', outdir,
-                       '-O', realfile]
-    except IndexError:
-        raise UTAHException('Cannot unpack image: ' + image)
-    if logmethod is not None:
-        logmethod('bsdtar extract command: ' + ' '.join(cmd))
-    return cmd
-
-
-def extract(image, filename, outdir=None, outfile=None, logmethod=None, **kw):
-    """
-    Return a subprocess to extract a file from an iso, passing any additional
-    keyword arguments to the subprocess.
-    If outfile is None, extract to standard out.
-    If outdir is present and outfile is None, extract the file to outdir with
-    the same name.
-    If outdir and outfile are both present, extract to outfile in outdir.
-    If the file is a hardlink, extract the actual file.
-    If the file is a symlink, return a command to create the symlink.
-    """
-    if outfile is None:
-        if outdir is None:
-            cmd = getrealfile(image, filename, logmethod=logmethod)
-            # The original function was designed to send the file over stdout,
-            # this removes that
-            # I think the updated function takes care of this already
-            # if '-C' in cmd:
-            #     cmd.remove('-C')
-            if cmd[-1] != filename:
-                if logmethod is not None:
-                    logmethod(filename + ' appears to be a link to ' + cmd[-1])
-                dirname = os.path.dirname(filename)
-                if not os.path.isdir(dirname):
-                    os.makedirs(dirname)
-                os.chmod(dirname, 0775)
-                if os.path.isfile(filename):
-                    os.chmod(filename, 0775)
-                return subprocess.call(cmd, stdout=open(filename, 'w'), **kw)
-        else:
-            cmd = getrealfile(image, filename,
-                              outdir=outdir, logmethod=logmethod)
-        if '-O' in cmd:
-            cmd.remove('-O')
-        return subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE, **kw).communicate()
-    else:
-        cmd = getrealfile(image, filename, logmethod=logmethod)
-        return subprocess.call(cmd, stdout=open(outfile, 'w'), **kw)
-
-
-def dump(image, filename, logmethod=None, **kw):
-    """
-    Return a subprocess dumping a file in an iso to standard out, passing any
-    additional keyword arguments to the subprocess.
-    """
-    cmd = getrealfile(image, filename, logmethod=logmethod)
-    return subprocess.Popen(cmd, stdout=subprocess.PIPE, **kw)
-
-
 class ISO(object):
     """
     Provide a simplified method of interfacing with images.
@@ -226,18 +127,88 @@
         self.logger.debug("%s read, %s%% of %s total" % (read, percent, total))
 
     def listfiles(self, returnlist=False):
-        return listfiles(self.image, self.logger.debug, returnlist=returnlist)
+        """
+        Return either a subprocess instance listing the contents of an iso, or
+        a list of files in the iso if returnlist is True.
+        """
+        cmd = ['bsdtar', '-t', '-f', self.image]
+        self.logger.debug('bsdtar list command: ' + ' '.join(cmd))
+        if returnlist:
+            return subprocess.check_output(cmd).strip().split('\n')
+        else:
+            return subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE).communicate()
 
     def getrealfile(self, filename, outdir=None):
-        return getrealfile(self.image, filename, outdir=outdir,
-                           logmethod=self.logger.debug)
+        """
+        Return a command to safely extract a file from an iso.
+        Based on unbsdtar-safelink from ubuntu iso testing.
+        """
+        cmd = ['bsdtar', '-t', '-v', '-f', self.image, filename]
+        self.logger.debug('bsdtar list command: ' + ' '.join(cmd))
+        output = (subprocess.Popen(cmd, stdout=subprocess.PIPE)
+                  .communicate()[0].split('\n')[0].split())
+        try:
+            if output[0].startswith('l'):
+                if outdir is None:
+                    cmd = ['ln', '-s', output[-1], output[-3]]
+                else:
+                    cmd = ['ln', '-s', output[-1],
+                           os.path.join(outdir, output[-3])]
+            else:
+                realfile = output[-1]
+                if outdir is None:
+                    cmd = ['bsdtar', '-x', '-f', self.image, '-O', realfile]
+                else:
+                    cmd = ['bsdtar', '-x', '-f', self.image, '-C', outdir,
+                           '-O', realfile]
+        except IndexError:
+            raise UTAHException('Cannot unpack image: ' + self.image)
+        self.logger.debug('bsdtar extract command: ' + ' '.join(cmd))
+        return cmd
 
     def extract(self, filename, outdir=None, outfile=None, **kw):
-        return extract(self.image, filename, outdir, outfile,
-                       logmethod=self.logger.debug, **kw)
+        """
+        Return a subprocess to extract a file from an iso, passing any
+        additional keyword arguments to the subprocess.
+        If outfile is None, extract to standard out.
+        If outdir is present and outfile is None, extract the file to outdir
+        with the same name.
+        If outdir and outfile are both present, extract to outfile in outdir.
+        If the file is a hardlink, extract the actual file.
+        If the file is a symlink, return a command to create the symlink.
+        """
+        if outfile is None:
+            if outdir is None:
+                cmd = self.getrealfile(filename)
+                if cmd[-1] != filename:
+                    self.logger.debug(filename + ' appears to be a link to '
+                                      + cmd[-1])
+                    dirname = os.path.dirname(filename)
+                    if not os.path.isdir(dirname):
+                        os.makedirs(dirname)
+                    os.chmod(dirname, 0775)
+                    if os.path.isfile(filename):
+                        os.chmod(filename, 0775)
+                    return subprocess.call(cmd, stdout=open(filename, 'w'),
+                                           **kw)
+            else:
+                cmd = self.getrealfile(filename, outdir=outdir)
+            if '-O' in cmd:
+                cmd.remove('-O')
+            return subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE, **kw).communicate()
+        else:
+            cmd = self.getrealfile(filename)
+            return subprocess.call(cmd, stdout=open(outfile, 'w'), **kw)
 
     def dump(self, filename, **kw):
-        return dump(self.image, filename, logmethod=self.logger.debug, **kw)
+        """
+        Return a subprocess dumping a file in an iso to standard out, passing
+        any additional keyword arguments to the subprocess.
+        """
+        cmd = self.getrealfile(filename)
+        return subprocess.Popen(cmd, stdout=subprocess.PIPE, **kw)
 
     def extractall(self):
         for myfile in self.listfiles(returnlist=True):

=== modified file 'utah/isotest/iso_static_validation.py'
--- utah/isotest/iso_static_validation.py	2012-07-24 11:07:42 +0000
+++ utah/isotest/iso_static_validation.py	2012-11-14 16:06:21 +0000
@@ -34,7 +34,8 @@
 
 lib_path = os.path.abspath('../')
 sys.path.append(lib_path)
-import iso
+from utah.iso import ISO
+from utah import config
 
 
 # Defaults
@@ -55,18 +56,26 @@
     print 'The usage:', sys.argv[0],  '--name release-variant-arch.iso'
     sys.exit(1)
 else:
-    st_name = args.name.rstrip('.iso')
-    try:
-        (st_release, st_variant, st_arch) = st_name.rsplit('-')
-    except ValueError:
-        sys.exit("Image name be in the form of release-variant-arch.iso")
-    logging.debug("The file being tested is: '%s'", args.name)
+    # Set default path
+    iso_location = args.name
+    # Iterate over possible paths, starting with old default behavior
+    for path in (os.path.join(DEFAULT_ISOROOT, args.name), args.name):
+        if os.path.isfile(path):
+            iso_location = path
+            break
+    iso = ISO(iso_location)
+    st_variant = iso.installtype
+    st_arch = iso.arch
+
 
 #Set a flag for ubiquity based images
 if st_variant == 'desktop' or st_variant == 'dvd':
     ubiquity_image = True
 else:
     ubiquity_image = False
+    
+#Setup the path for data files
+DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')
 
 
 class TestValidateISO(unittest.TestCase):
@@ -79,12 +88,16 @@
             unittest.fail("Setup error")
 
     def setUp(self):
-        self.iso_name = args.name
         self.block_size = ONE_MB_BLOCK
-        self.iso_location = os.path.join(DEFAULT_ISOROOT, args.name)
-        self.st_release = st_release
-        self.st_variant = st_variant
-        self.st_arch = st_arch
+        self.iso_location = iso_location
+        logging.debug('Using iso at: ' + self.iso_location)
+        self.iso = ISO(self.iso_location, logger=logging)
+        self.st_release = self.iso.series
+        self.st_variant = self.iso.installtype
+        self.st_arch = self.iso.arch
+        self.iso_name = '-'.join([self.st_release, self.st_variant,
+                                  self.st_arch]) + '.iso'
+        logging.debug('Standard name for this iso is: ' + self.iso_name)
 
         if self.st_release != 'precise':
             self.url = DEFAULT_URL
@@ -164,7 +177,7 @@
             self.fail("Failed to fetch files list from the server")
 
         logging.debug('Extracting list file info from ISO')
-        (stdout, stderr) = iso.listfiles(self.iso_location)
+        (stdout, stderr) = self.iso.listfiles()
         logging.debug('Checking for error in extracting the file list')
         self.assertEqual(stderr, '')
 
@@ -188,9 +201,8 @@
             self.fail("Failed to fetch manifest file from the server")
 
         logging.debug('Extracting manifest from the iso')
-        (stdout, stderr) = iso.extract(self.iso_location,
-                                       'casper/filesystem.manifest',
-                                       self.temp_dir)
+        (stdout, stderr) = self.iso.extract('casper/filesystem.manifest',
+                                            self.temp_dir)
         logging.debug('Check for error in extracting manifest info from iso')
         self.assertEqual(stderr, '')
 
@@ -206,8 +218,7 @@
     # the same as that given in distro info
     def test_build_id(self):
         logging.debug('Extract distro information of the image from the iso')
-        (stdout, stderr) = iso.extract(self.iso_location, "./.disk/info",
-                                       self.temp_dir)
+        (stdout, stderr) = self.iso.extract("./.disk/info", self.temp_dir)
         logging.debug('Checking for error in extracting distro info from iso')
         self.assertEqual(stderr, '')
 
@@ -237,8 +248,7 @@
                          st_arch != 'powerpc', "Skipping non ubiquity images")
     def test_wubi(self):
         logging.debug('Extracting wubi from the desktop images')
-        (stdout, stderr) = iso.extract(self.iso_location, './wubi.exe',
-                                       self.temp_dir)
+        (stdout, stderr) = self.iso.extract('./wubi.exe', self.temp_dir)
         logging.debug('Check for error in extracting wubi.exe from the iso')
         self.assertEqual(stderr, '')
 
@@ -256,11 +266,10 @@
     @unittest.skipUnless(st_variant == 'desktop' and
                          st_arch != 'powerpc', "Skipping for non desktop iso")
     def test_files_ubiquity(self):
-        (stdout, stderr) = iso.listfiles(self.iso_location,
-                                         logmethod=logging.debug)
+        (stdout, stderr) = self.iso.listfiles()
         logging.debug('Check for error in extracting file list from the iso')
         self.assertEqual(stderr, '')
-        files_list = open("data/file_list_ubiquity")
+        files_list = open(os.path.join(DATA_PATH, 'file_list_ubiquity'))
         for list_server in files_list:
             logging.debug('Check if relevant files are present in the iso')
             self.assertIn(list_server.rstrip(), stdout)
@@ -269,11 +278,10 @@
     @unittest.skipUnless(st_variant == 'dvd',
                         "This file list test is only specific for dvd")
     def test_files_ubiquity_dvd(self):
-        (stdout, stderr) = iso.listfiles(self.iso_location,
-                                         logmethod=logging.debug)
+        (stdout, stderr) = self.iso.listfiles()
         logging.debug('Check for error in extracting file list from the iso')
         self.assertEqual(stderr, '')
-        files_list = open("data/file_list_ubiquity_dvd")
+        files_list = open(os.path.join(DATA_PATH, 'file_list_ubiquity_dvd'))
         for list_entry in files_list:
             logging.debug('Check if relevant files are present in the iso')
             self.assertIn(list_entry.rstrip(), stdout)
@@ -285,15 +293,14 @@
     # Test is skipped for ubiquity based images
     @unittest.skipIf(ubiquity_image, "Skipping for ubiquity images")
     def test_files_di(self):
-        (stdout, stderr) = iso.listfiles(self.iso_location,
-                                         logmethod=logging.debug)
+        (stdout, stderr) = self.iso.listfiles()
         logging.debug('Check for error in extracting file list from the iso')
         self.assertEqual(stderr, '')
 
         if self.st_arch == 'powerpc' or self.st_arch == 'amd64+mac':
-            files_list = open("data/file_list_di_powerpc")
+            files_list = open(os.path.join(DATA_PATH, 'file_list_di_powerpc'))
         else:  # i386 or amd64
-            files_list = open("data/file_list_di")
+            files_list = open(os.path.join(DATA_PATH, 'file_list_di'))
 
         for list_server in files_list:
             logging.debug('check if important d-i files are present in iso')
@@ -303,8 +310,7 @@
     @unittest.skipUnless(ubiquity_image and st_arch != 'powerpc',
                          "vmlinuz is present only for ubiquity based images")
     def test_vmlinuz(self):
-        (stdout, stderr) = iso.extract(self.iso_location, 'casper/vmlinuz',
-                                       self.temp_dir)
+        (stdout, stderr) = self.iso.extract('casper/vmlinuz', self.temp_dir)
         logging.debug('Asserting that vmlinuz is present in the iso')
         self.assertEqual(stderr, '')
 
@@ -323,9 +329,8 @@
     @unittest.skipUnless(ubiquity_image, "Skipping for non ubiquity images")
     def test_filesystem_squashfs(self):
         logging.debug('Extracting the filesystem.squashfs')
-        (stdout, stderr) = iso.extract(self.iso_location,
-                                       'casper/filesystem.squashfs',
-                                       self.temp_dir)
+        (stdout, stderr) = self.iso.extract('casper/filesystem.squashfs',
+                                            self.temp_dir)
         logging.debug('Check for error in extracting filesystem.squashfs')
         self.assertEqual(stderr, '')