Merge lp:~mwhudson/utah/live-server into lp:utah

[Michael Hudson-Doyle]

I tested this branch with something approximating:

$ echo '{"group": "mwhudson", "launchpad_user": "mwhudson"}' > ~/.utah/config
$ wget http://people.canonical.com/~mwh/live-server-for-utah.iso -O ~/isos/live-server-for-utah.iso
$ git clone https://github.com/CanonicalLtd/subiquity ~/src/subiquity
$ bzr branch lp:ubuntu-test-cases/server ~/src/ubuntu-test-cases/server
$ PYTHONPATH=. ./examples/run_utah_tests.py -m virtual-live-server ~/src/ubuntu-test-cases/server/runlists/lxc.run -p ~/src/subiquity/examples/answers.yaml -x /etc/utah/bridged-network-vm.xml -i ~/isos/live-server-for-utah.iso

Please let me know what you think. It's a bit rough but it should at least not risk breaking anything else.

[lp:~mwhudson/utah/live-server]

1111. By Joshua Powers on 2017-12-20

Enable live-server ISO testing

This allows for the testing of live-server ISOs (aka subiquity)
based ISOs.

=== modified file 'examples/run_utah_tests.py'
--- examples/run_utah_tests.py	2017-07-06 18:03:17 +0000
+++ examples/run_utah_tests.py	2017-12-20 00:09:54 +0000
@@ -72,6 +72,9 @@
                 'utah-baremetal package needed for physical machines')
         return get_baremetal(args.arch, **kw)
     else:
+        if args.machinetype == 'virtual-live-server':
+            from utah.provisioning.live_server import LiveServerVM
+            kw['machinetype'] = LiveServerVM
         if 'vm' in MISSING:
             raise UTAHException('Virtual Machine support is not available')
         # TODO: consider removing these as explicit command line arguments

=== modified file 'utah/config.py'
--- utah/config.py	2017-08-30 21:58:10 +0000
+++ utah/config.py	2017-12-20 00:09:54 +0000
@@ -214,6 +214,11 @@
                 'format': 'path',
                 'default': None,
             },
+            'launchpad_user': {
+                'description': 'Launchpad whose keys should be imported on a live-server install',
+                'type': 'string',
+                'default': 'server-team-bot',
+            },
             'logfile': {
                 'description': 'Path to log file',
                 'type': 'string',

=== modified file 'utah/parser.py'
--- utah/parser.py	2017-07-06 18:03:17 +0000
+++ utah/parser.py	2017-12-20 00:09:54 +0000
@@ -80,7 +80,7 @@
                 "\t\t/usr/share/utah/client/examples/master.run"),
         formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument('-m', '--machinetype', metavar='MACHINETYPE',
-                        choices=('physical', 'virtual'),
+                        choices=('physical', 'virtual', 'virtual-live-server'),
                         default=config.machinetype,
                         help='Type of machine to provision (%(choices)s) '
                              '(Default is %(default)s)')

=== added file 'utah/provisioning/live_server.py'
--- utah/provisioning/live_server.py	1970-01-01 00:00:00 +0000
+++ utah/provisioning/live_server.py	2017-12-20 00:09:54 +0000
@@ -0,0 +1,411 @@
+
+# Support for testing "live-server" (aka subiquity) images.  Missing
+# functionality: extracting syslog, installer, curtin logs from the
+# image file. Not sure how to hook that in.
+
+import os
+import re
+import select
+import string
+import shutil
+import tempfile
+import time
+
+import yaml
+
+import libvirt
+
+from xml.etree import ElementTree
+
+from utah.config import config
+
+from utah.process import ProcessRunner
+
+from utah.timeout import UTAHTimeout
+
+from utah.provisioning.ssh import SSHMixin
+from utah.provisioning.vm import (
+    LibvirtVM,
+    random_mac_address,
+    UTAHVMProvisioningException,
+    )
+
+
+class LiveServerVM(SSHMixin, LibvirtVM):
+    def __init__(self, machineid, diskbus, disksizes, emulator, name, **kw):
+        self.diskbus = diskbus
+        self.disksizes = disksizes
+        self.emulator = emulator
+        self.macs = config.macs
+        self.prefix = config.prefix
+        self.disks = []
+        if name is None:
+            name = '-'.join([str(config.prefix), str(machineid)])
+            autoname = True
+        super(LiveServerVM, self).__init__(
+            machineid=machineid, name=name, **kw)
+        if autoname:
+            self._namesetup()
+            self._loggerunsetup()
+            self._loggersetup()
+        if self.inventory is not None:
+            self.cleanfunction(self.inventory.destroy,
+                               machineid=self.machineid)
+        self.directory = os.path.join(config.vmpath, self.name)
+        self.cleanfile(self.directory)
+        self.logger.info('Checking if machine directory {} exists'
+                         .format(self.directory))
+        if not os.path.isdir(self.directory):
+            self.logger.debug('Creating {}'.format(self.directory))
+            try:
+                os.makedirs(self.directory)
+            except OSError as err:
+                raise UTAHVMProvisioningException(err)
+        self.logger.debug('LiveServerVM init finished')
+
+    def _tmpimage(self, image=None, tmpdir=None):
+        """Create a temporary copy of the image so libvirt will lock that one.
+
+        This allows other simultaneous processes to update the cached image.
+
+        :returns: The path to the temporary image
+        :rtype: str
+
+        """
+        if image is None:
+            image = self.image.image
+        if tmpdir is None:
+            tmpdir = self.tmpdir
+        self.logger.info('Making temp copy of install image')
+        tmpimage = os.path.join(tmpdir, os.path.basename(image))
+        self.logger.debug('Copying %s to %s', image, tmpimage)
+        shutil.copyfile(image, tmpimage)
+        return tmpimage
+
+    def _setuppreseed(self, tmpdir):
+        # Read the provided "answers file" for subiquity, overwrite
+        # the Identity section so that we can log in how SSHMixin
+        # expects to be able to and prepare a disk image in the form
+        # subiquity expects that contains it.
+        self.answers_image_path = os.path.join(tmpdir, "answers.img")
+        mount_dir = os.path.join(tmpdir, "answers")
+        os.makedirs(mount_dir)
+        cmds = [
+            ['truncate', '-s', '1M', self.answers_image_path],
+            ['mkfs.ext2', '-U', '00c629d6-06ab-4dfd-b21e-c3186f34105d', self.answers_image_path],
+            # XXX This probably should change to "sudo mount" rather
+            # than dodgy fuse packages from universe:
+            ['fuse-ext2', self.answers_image_path, mount_dir, '-o', 'rw+'],
+            ]
+        for cmd in cmds:
+            if ProcessRunner(cmd).returncode != 0:
+                raise UTAHVMProvisioningException(
+                    '{} failed'.format(cmd))
+        yaml_file = os.path.join(mount_dir, "answers.yaml")
+        answers = yaml.safe_load(open(self.preseed))
+        answers['Identity'] = {
+            # This is "ubuntu", crypted:
+            'password': '$6$wdAcoXrU039hKYPd$508Qvbe7ObUnxoj15DRCkzC3qO7edjH0VV7BPNRDYK4QR8ofJaEEF2heacn0QgD.f8pO8SNp83XNdWG6tocBM1',
+            'hostname': self.name,
+            'realname': 'Utah',
+            'ssh-import-id': 'lp:' + config.launchpad_user.encode('utf-8'),
+            'username': config.user,
+            }
+
+        with open(yaml_file, 'w') as f:
+            f.write(yaml.dump(answers))
+        cmd = ['fusermount', '-u', mount_dir]
+        if ProcessRunner(cmd).returncode != 0:
+            raise UTAHVMProvisioningException(
+                '{} failed'.format(cmd))
+        disk = {'bus': self.diskbus,
+                'file': self.answers_image_path,
+                'size': 1024*1024,
+                'type': 'raw'}
+        self.disks.append(disk)
+
+    def _createdisks(self, disksizes=None):
+        """Create disk files if needed and build a list of them."""
+        self.logger.info('Creating disks')
+        if disksizes is None:
+            disksizes = self.disksizes
+        for index, size in enumerate(disksizes):
+            disksize = '{}G'.format(size)
+            basename = 'disk{}.qcow2'.format(index)
+            diskfile = os.path.join(self.directory, basename)
+            if not os.path.isfile(diskfile):
+                cmd = ['qemu-img', 'create', '-f', 'qcow2', diskfile,
+                       disksize]
+                self.logger.debug('Creating %s disk using: %s',
+                                  disksize, ' '.join(cmd))
+                if ProcessRunner(cmd).returncode != 0:
+                    raise UTAHVMProvisioningException(
+                        'Could not create disk image at {}'.format(diskfile))
+            disk = {'bus': self.diskbus,
+                    'file': diskfile,
+                    'size': disksize,
+                    'type': 'qcow2'}
+            self.disks.append(disk)
+            self.cleanfile(diskfile)
+            self.logger.debug('Adding disk to list')
+
+    def _installxml_rewrite_boot(self, xmlt):
+        # Set the arch appropriately, remove any configuration of
+        # kernel/initrd/commandline and configure the VM to boot from
+        # CD.
+        xmlt.getroot().set('type', 'kvm')
+        ose = xmlt.find('os')
+        if self.image.arch == 'i386':
+            ose.find('type').set('arch', 'i686')
+        elif self.image.arch == 'amd64':
+            ose.find('type').set('arch', 'x86_64')
+        elif self.image.arch == 'ppc64el':
+            ose.find('type').set('arch', 'ppc64le')
+        else:
+            ose.find('type').set('arch', self.image.arch)
+        self.logger.debug('Setting up boot info')
+        for kernele in list(ose.iterfind('kernel')):
+            ose.remove(kernele)
+        for initrde in list(ose.iterfind('initrd')):
+            ose.remove(initrde)
+        for cmdlinee in list(ose.iterfind('cmdline')):
+            ose.remove(cmdlinee)
+        boot = ElementTree.Element('boot')
+        boot.set('dev', 'cdrom')
+        ose.append(boot)
+
+    def _installxml_rewrite_disks(self, devices, image):
+        # 1. Remove all existing disks
+        # 2. Hook image up to cdrom
+        # 3. Create all required disks.
+        self.logger.debug('Setting up disks')
+        for disk in list(devices.iterfind('disk')):
+            if disk.get('device') == 'disk':
+                devices.remove(disk)
+                self.logger.debug('Removed existing disk')
+            #TODO: Add a cdrom if none exists
+            if disk.get('device') == 'cdrom':
+                if disk.find('source') is not None:
+                    disk.find('source').set('file', image)
+                    self.logger.debug('Rewrote existing CD-ROM')
+                else:
+                    source = ElementTree.Element('source')
+                    source.set('file', image)
+                    disk.append(source)
+                    self.logger.debug(
+                        'Added source to existing CD-ROM')
+        for disk in self.disks:
+            diske = ElementTree.Element('disk')
+            diske.set('type', 'file')
+            diske.set('device', 'disk')
+            driver = ElementTree.Element('driver')
+            driver.set('name', 'qemu')
+            driver.set('type', disk['type'])
+            diske.append(driver)
+            serial = ElementTree.Element('serial')
+            serial.text = "disk-{}".format(self.disks.index(disk))
+            diske.append(serial)
+            source = ElementTree.Element('source')
+            source.set('file', disk['file'])
+            diske.append(source)
+            target = ElementTree.Element('target')
+            dev = ("vd{}"
+                   .format(string.ascii_lowercase[self.disks.index(disk)]))
+            target.set('dev', dev)
+            target.set('bus', disk['bus'])
+            diske.append(target)
+            devices.append(diske)
+            self.logger.debug('Added %s disk', str(disk['size']))
+
+    def _installxml_rewrite_nics(self, devices):
+        macs = list(self.macs)
+        for interface in devices.iterfind('interface'):
+            if interface.get('type') in ['network', 'bridge']:
+                if len(macs) > 0:
+                    mac = macs.pop(0)
+                    interface.find('mac').set('address', mac)
+                    self.logger.debug(
+                        'Rewrote interface to use specified mac address: %s',
+                        mac)
+                else:
+                    mac = random_mac_address()
+                    interface.find('mac').set('address', mac)
+                    self.macs.append(mac)
+                    self.logger.debug(
+                        'Rewrote interface to use random mac address: %s',
+                        mac)
+            if interface.get('type') == 'bridge':
+                interface.find('source').set('bridge', config.bridge)
+
+    def _installxml_add_serial(self, devices):
+        serial = ElementTree.Element('serial')
+        serial.set('type', 'file')
+        source = ElementTree.Element('source')
+        source.set('path', self.syslog)
+        #TODO: reintegrate this when the required qemu version lands
+#        source.set('append', 'on')  # lp#1548499
+        serial.append(source)
+        target = ElementTree.Element('target')
+        target.set('port', '0')
+        serial.append(target)
+        devices.append(serial)
+
+    def _installxml(self, image, tmpdir, xml):
+        """Return the XML tree to be passed to libvirt for VM installation."""
+        self.logger.info('Creating installation XML')
+        xmlt = ElementTree.ElementTree(file=xml)
+        xmlt.find('on_reboot').text = 'destroy'
+        self.logger.debug('Rewriting basic info')
+        xmlt.find('name').text = self.name
+        xmlt.find('uuid').text = self.uuid
+        self._installxml_rewrite_boot(xmlt)
+        devices = xmlt.find('devices')
+        self._installxml_rewrite_disks(devices, image)
+        self._installxml_rewrite_nics(devices)
+        self._installxml_add_serial(devices)
+        xmlt.write(os.path.join(tmpdir, 'install.xml'))
+        self.logger.info('Installation XML ready')
+        return xmlt
+
+    def _installvm(self, lv=None, tmpdir=None, xml=None):
+        """Install a VM, then undefine it in libvirt.
+
+        The final installation will recreate the VM using the existing disks.
+
+        """
+        self.logger.info('Creating VM')
+        if lv is None:
+            lv = self.lv
+        if xml is None:
+            xml = self.xml
+        if tmpdir is None:
+            tmpdir = self.tmpdir
+        os.chmod(tmpdir, 0o755)
+        self.logger.info(
+            'Installing system on VM (may take over an hour)')
+        self.logger.info('You can watch the progress with virt-viewer')
+        log_filename = os.path.join(config.logpath,
+                                    '{}.syslog.log'.format(self.name))
+        self.logger.info('Logs will be written to %s', log_filename)
+
+        before = time.time()
+        try:
+            vm = lv.defineXML(ElementTree.tostring(xml.getroot()))
+            vm.create()
+            while vm.isActive() is not 0:
+                time.sleep(1)
+        finally: # This stops us from leaking VMs on exceptions
+            try:
+                vm.destroy()
+            except libvirt.libvirtError:
+                pass
+            finally:
+                vm.undefine()
+
+        after = time.time()
+        self.logger.info('Installation complete in %s seconds' %
+                         int(after-before))
+    def _finalxml(self, tmpdir, xml):
+        """Create the XML to be used for the post-installation VM.
+
+        This "ejects" the CD and removes the answers image file.
+        """
+        self.logger.info('Creating final VM XML')
+        self.logger.debug('Setting VM to reboot normally on reboot')
+        xml.find('on_reboot').text = 'restart'
+        self.logger.debug('Removing VM install parameters')
+        devices = xml.find('devices')
+        for disk in list(devices.iterfind('disk')):
+            if disk.get('device') == 'cdrom':
+                disk.remove(disk.find('source'))
+            source = disk.find('source')
+            if source is not None and source.get('file') == self.answers_image_path:
+                devices.remove(disk)
+        xml.write(os.path.join(tmpdir, 'final.xml'))
+        return xml
+
+    def _create(self, provision_data):
+        """Create the VM, install the system, and prepare it to boot.
+        """
+        self.logger.info('Creating custom virtual machine')
+
+        tmpdir = tempfile.mkdtemp(prefix='{}/tmp'.format(self.directory))
+        self.cleanfile(tmpdir)
+        self.logger.debug('Working dir: %s', tmpdir)
+        os.chdir(tmpdir)
+
+        if provision_data:
+            1/0
+
+        self._setuppreseed(tmpdir=tmpdir)
+
+        #self._setuplogging(tmpdir=tmpdir)
+
+        self._createdisks()
+
+        image = self._tmpimage(image=self.image.image, tmpdir=tmpdir)
+
+        xml = self._installxml(image=image, tmpdir=tmpdir, xml=self.xml)
+
+        self._installvm(lv=self.lv, tmpdir=tmpdir, xml=xml)
+
+        xml = self._finalxml(tmpdir=tmpdir, xml=xml)
+
+        self.logger.info('Setting up final VM')
+        self.vm = self.lv.defineXML(ElementTree.tostring(xml.getroot()))
+        self.cleanfunction(self.vm.destroy)
+        self.cleanfunction(self.vm.undefine)
+        if self.poweroff:
+            self.cleanfunction(self.vm.shutdown, force=True)
+
+    def _start(self):
+        """Start the VM."""
+        self.logger.info('Starting CustomVM')
+        if self.vm is not None:
+            if self.vm.isActive() == 0:
+                self.vm.create()
+        else:
+            raise UTAHVMProvisioningException('Failed to provision VM')
+
+        try:
+            self.logger.info(
+                'Waiting %d seconds for ping response', config.boottimeout)
+            self.pingpoll(timeout=config.boottimeout)
+        except UTAHTimeout:
+            # Ignore timeout for ping, since depending on the network
+            # configuration ssh might still work despite of the ping failure.
+            self.logger.warning('Network connectivity (ping) failure')
+        self.sshpoll(timeout=config.boottimeout)
+        self._enable_root_login()
+        self.active = True
+
+    def _enable_root_login(self):
+        try:
+            self.ssh_client.connect(self.name,
+                                    username=config.user,
+                                    key_filename=config.sshprivatekey)
+            chan = self.ssh_client.get_transport().open_session()
+            chan.get_pty()
+            chan.set_combine_stderr(True)
+            chan.exec_command("sudo su -c 'cp -aT .ssh /root/.ssh && chown root:root -R /root/.ssh && echo ok'")
+            def expect(pat):
+                prog = re.compile(pat)
+                buf = ''
+                while True:
+                    match = prog.match(buf)
+                    if match:
+                        return match
+                    r, _, _ = select.select([chan], [], [], 10)
+                    if not r:
+                        raise Exception("no output for 10s after %r", buf)
+                    c = chan.recv(1)
+                    buf += c
+            expect('\[sudo\] password for ([a-z]+): ')
+            chan.send('ubuntu\n')
+            expect('\r\n')
+            m = expect("Sorry|ok")
+            if m.string != "ok":
+                raise UTAHVMProvisioningException("_enable_root_login failed")
+        finally:
+            self.ssh_logger.debug('Closing SSH connection')
+            self.ssh_client.close()