Merge lp:~heber013/utah/adding-setup-for-autopkgtest into lp:utah

[Heber Parrucci]

* Creating a serial console on ttyS1 for autopkgtest to use it.
* Adding option to poweroff VM after tests have run.

Creating a serial console on ttyS1 for autopkgtest to use it.
Adding option to poweroff VM after tests have run.

[Jean-Baptiste Lallement (jibel)]

Approved. Thanks.

review: Approve

=== modified file 'examples/run_utah_tests.py'
--- examples/run_utah_tests.py	2013-06-20 00:06:02 +0000
+++ examples/run_utah_tests.py	2017-07-06 18:08:08 +0000
@@ -54,6 +54,7 @@
     if image is None:
         image = ISO(arch=args.arch, installtype=args.type, series=args.series)
     kw = {'clean': (not args.no_destroy),
+          'poweroff': args.poweroff,
           'boot': args.boot,
           'debug': args.debug,
           'image': image,

=== modified file 'setup.py'
--- setup.py	2013-06-19 20:30:45 +0000
+++ setup.py	2017-07-06 18:08:08 +0000
@@ -52,6 +52,9 @@
               'utah.client.probe',
               'utah.provisioning',
               'utah.provisioning.baremetal'],
+    package_data={
+        'utah': ['provisioning/qemu-scripts/*']
+    },
     maintainer=maintainer,
     maintainer_email=maintainer_email,
 )

=== modified file 'templates/casper-preseed-script.jinja2'
--- templates/casper-preseed-script.jinja2	2013-05-22 16:56:32 +0000
+++ templates/casper-preseed-script.jinja2	2017-07-06 18:08:08 +0000
@@ -6,3 +6,4 @@
 cp utah-copy-files.sh /root/
 cp -r utah-copy-files /root/
 cp -r utah-autorun /root/
+cp qemu* /root

=== modified file 'templates/utah-latecommand.jinja2'
--- templates/utah-latecommand.jinja2	2016-06-10 17:40:15 +0000
+++ templates/utah-latecommand.jinja2	2017-07-06 18:08:08 +0000
@@ -98,9 +98,20 @@
     fi
 }
 
+qemu_setup() {
+    if [ -f qemu-setup.py ] ; then
+        cp qemu-setup.py /target/usr/local/bin/qemu-setup.py
+        chmod 755 /target/usr/local/bin/qemu-setup.py
+        cp qemu-setup.service /target/lib/systemd/system/qemu-setup.service
+        chmod 644 /target/lib/systemd/system/qemu-setup.service
+        chroot /target /bin/systemctl enable qemu-setup.service
+    fi
+}
+
 copy_ssh_keys
 write_utah_data
 autologin
 autorun
 copy_provisioned_files
 copy_rsyslog_cfg   # do this last to avoid getting each syslog msg 2 times
+qemu_setup

=== modified file 'utah/config.py'
--- utah/config.py	2017-06-06 20:18:58 +0000
+++ utah/config.py	2017-07-06 18:08:08 +0000
@@ -81,6 +81,12 @@
                 'type': 'boolean',
                 'default': True,
             },
+            'poweroff': {
+                'description': (
+                    'Setting for whether to power off VM after a test run'),
+                'type': 'boolean',
+                'default': False,
+            },
             'client_install_timeout': {
                 'description': (
                     'Maximum amount of time to wait '

=== modified file 'utah/parser.py'
--- utah/parser.py	2013-05-28 11:58:47 +0000
+++ utah/parser.py	2017-07-06 18:08:08 +0000
@@ -110,6 +110,8 @@
                              '(%(choices)s) (Default is %(default)s)')
     parser.add_argument('-n', '--no-destroy', action='store_true',
                         help='Preserve VM after tests have run')
+    parser.add_argument('--poweroff', action='store_true',
+                        help='Power off VM after tests have run')
     parser.add_argument('-d', '--debug', action='store_true',
                         help='Enable debug logging')
     parser.add_argument('-j', '--json', action='store_true',

=== modified file 'utah/provisioning/provisioning.py'
--- utah/provisioning/provisioning.py	2017-03-03 14:35:49 +0000
+++ utah/provisioning/provisioning.py	2017-07-06 18:08:08 +0000
@@ -24,6 +24,7 @@
 import pipes
 import re
 import shutil
+import stat
 import sys
 import urllib
 import uuid
@@ -63,8 +64,8 @@
 
     """
 
-    def __init__(self, boot=config.boot, clean=True, debug=False,
-                 image=None, initrd=config.initrd, inventory=None,
+    def __init__(self, boot=config.boot, clean=True, poweroff=False,
+                 debug=False, image=None, initrd=config.initrd, inventory=None,
                  kernel=config.kernel, machineid=config.machineid,
                  machineuuid=config.machineuuid, name=config.name, new=False,
                  preseed=config.preseed, rewrite=config.rewrite,
@@ -110,6 +111,7 @@
         # TODO: Consider a global temp file creator, maybe as part of install.
         self.boot = boot
         self.clean = clean
+        self.poweroff = poweroff
         self.debug = debug
         self.image = image
         self.inventory = inventory
@@ -522,11 +524,13 @@
         if self.clean:
             cleanup.add_path(path)
 
-    def cleanfunction(self, function, *args, **kw):
+    def cleanfunction(self, function, force=False, *args, **kw):
         """Register a function to be run on cleanup.
 
         :param function: A callable that will do some cleanup.
         :type function: callable
+        :param force: whether to force adding the cleanup function.
+        :type force: boolean
         :param args: Positional arguments to the function.
         :type args: Tuple
         :param kw: Keyword arguments to the function.
@@ -536,7 +540,7 @@
 
         """
         # TODO: consider doing this directly instead of through Machine
-        if self.clean:
+        if self.clean or force:
             cleanup.add_function(60, function, *args, **kw)
 
     def cleancommand(self, cmd):
@@ -703,6 +707,13 @@
                        filename,
                        log_file='/var/log/utah-install')
 
+    def _copy_qemu_scripts(self, tmpdir=None):
+        initrd_work_dir = os.path.join(tmpdir, 'initrd')
+        if not os.path.exists(initrd_work_dir):
+            os.makedirs(initrd_work_dir)
+        # copy the qemu scripts into initrd
+        self.copy_qemu_scripts_to_path(initrd_work_dir)
+
     def _setuppreseed(self, tmpdir=None):
         """Rewrite the preseed to automate installation and access."""
         # TODO: document this better
@@ -978,3 +989,21 @@
         """
         iface = netifaces.ifaddresses(ifname)
         return iface[netifaces.AF_INET][0]['addr']
+
+    def get_qemu_scripts_dir(self):
+        """Return path of the source directory containing qemu setup scripts."""
+        src_dir = os.path.dirname(os.path.realpath(__file__))
+        return os.path.join(src_dir, 'qemu-scripts')
+
+    def copy_qemu_scripts_to_path(self, path):
+        """Copy all qemu scripts to required path and ensure correct permissions.
+        :param path: Target path to copy scripts to.
+        """
+        src_dir = self.get_qemu_scripts_dir()
+        mask = (stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH |
+                stat.S_IXOTH)
+        for src_name in os.listdir(src_dir):
+            src = os.path.join(src_dir, src_name)
+            dst = os.path.join(path, src_name)
+            shutil.copy2(src, dst)
+            os.chmod(dst, mask)

=== added directory 'utah/provisioning/qemu-scripts'
=== added file 'utah/provisioning/qemu-scripts/qemu-setup.py'
--- utah/provisioning/qemu-scripts/qemu-setup.py	1970-01-01 00:00:00 +0000
+++ utah/provisioning/qemu-scripts/qemu-setup.py	2017-07-06 18:08:08 +0000
@@ -0,0 +1,211 @@
+# Ubuntu Testing Automation Harness
+# Copyright 2017 Canonical Ltd.
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import subprocess
+import time
+
+AUTOPKGTEST_NAME = 'autopkgtest.service'
+AUTOPKGTEST_SERVICE = os.path.join('/lib/systemd/system', AUTOPKGTEST_NAME)
+AUTOPKGTEST_CFG = '/etc/default/grub.d/90-autopkgtest.cfg'
+
+""" This script is run on first boot of the qemu after installation, by the
+    qemu-setup.service. It will prepare the system by:
+    - Creating a serial console on ttyS1 that will be used in future by
+      autopkgtest. The service will be started automatically on all subsequent
+      boots by autopkgtest.service.
+    - Disable qemu-setup.service to stop this script from running on subsequent
+      boots.
+    - Power down the system to complete the setup process.
+"""
+
+
+def wait_for_network():
+    """Wait until network connection is active."""
+    max_count = 60
+    for count in range(max_count):
+        status = subprocess.call(['wget', '-q', '--spider', 'launchpad.net'])
+        if status == 0:
+            break
+        elif count >= max_count - 1:
+            raise Exception('No network connection available')
+        else:
+            print('wget attempt {} failed, trying again...'.format(count + 1))
+            time.sleep(1)
+
+
+def wait_for_locks():
+    """Wait for file locks to be released for doing install operations."""
+    max_count = 60
+    lock_files = [
+        '/var/lib/dpkg/lock',
+        '/var/cache/apt/archives/lock',
+        '/var/lib/apt/lists/lock',
+    ]
+    for count in range(max_count):
+        status = subprocess.call(['fuser'] + lock_files)
+        if status == 1:
+            break
+        elif count >= max_count - 1:
+            raise RuntimeError('File locks not released')
+        else:
+            print('file locks active, trying again...'.format(count + 1))
+            time.sleep(1)
+
+
+def get_architecture():
+    """Return architecture of running system."""
+    return subprocess.check_output(
+        ['dpkg', '--print-architecture']).decode().strip()
+
+
+def install_packages(packages):
+    """Install specfied list of packages."""
+    wait_for_locks()
+    subprocess.check_call(['dpkg', '--configure', '-a'])
+    subprocess.check_call(['apt-get', 'install', '-y'] + packages)
+
+
+def create_file(path, content):
+    """Create a file with specified path and content."""
+    with open(path, 'w') as f:
+        f.write(content)
+
+
+def setup_root_shell_service():
+    """Setup root shell service on ttyS1 for use by autopkgtest."""
+    create_file(
+        AUTOPKGTEST_SERVICE,
+        '[Unit]\n'
+        'Description=autopkgtest root shell on ttyS1\n'
+        'ConditionPathExists=/dev/ttyS1\n\n'
+        '[Service]\n'
+        'ExecStart=/bin/sh\n'
+        'StandardInput=tty-fail\n'
+        'StandardOutput=tty\n'
+        'StandardError=tty\n'
+        'TTYPath=/dev/ttyS1\n'
+        'SendSIGHUP=yes\n'
+        'SuccessExitStatus=0 208 SIGHUP SIGINT SIGTERM SIGPIPE\n\n'
+        '[Install]\n'
+        'WantedBy=multi-user.target\n'
+    )
+    os.chmod(AUTOPKGTEST_SERVICE, 0o644)
+    subprocess.check_call(['systemctl', 'enable', AUTOPKGTEST_NAME])
+
+
+def _grubd_autopkgtest_setup(architecture):
+    """Add autopkgtest config to grub.d directory.
+    :param architecture: Required architecture.
+    :return: True if changes made, False otherwise.
+    """
+    if architecture == 'i386':
+        create_file(
+            AUTOPKGTEST_CFG,
+            'GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0 vmalloc=512M"'
+        )
+        return True
+    elif architecture == 'amd64':
+        create_file(
+            AUTOPKGTEST_CFG,
+            'GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0"'
+        )
+        return True
+    return False
+
+
+def _grub_autopkgtest_setup(architecture):
+    """Add autopkgtest config to grub directory.
+    :param architecture: Required architecture.
+    :return: True if changes made, False otherwise.
+    """
+    changed = False
+    if architecture == 'i386':
+        subprocess.check_call(
+            ['sed', '-i',
+             '/CMDLINE_LINUX_DEFAULT/ s/"$/ console=ttyS0 '
+             'vmalloc=512M"/', '/etc/default/grub']
+        )
+        changed = True
+    elif architecture == 'amd64':
+        subprocess.check_call(
+            ['sed', '-i',
+             '/CMDLINE_LINUX_DEFAULT/ s/"$/ console=ttyS0"/',
+             '/etc/default/grub']
+        )
+        changed = True
+
+    zero_timeout = subprocess.check_output(
+        ['grep', '-q', 'GRUB_HIDDEN_TIMEOUT=0', '/etc/default/grub']
+    ) == 0
+
+    if zero_timeout:
+        subprocess.check_output(
+            ['sed', '-i', '/^GRUB_TIMEOUT=/ s/=.*$/=1/',
+             '/etc/default/grub']
+        )
+        changed = True
+
+    return changed
+
+
+def setup_serial_console():
+    """Setup a serial console for autopkgtest to use."""
+    # This method is derived from:
+    # /usr/share/autopkgtest/setup-commands/setup-testbed
+    setup_root_shell_service()
+    grub = subprocess.call(['which', 'update-grub']) == 0
+    if not os.path.isfile(AUTOPKGTEST_CFG) and grub:
+        arch = get_architecture()
+        if os.path.isdir('/etc/default/grub.d'):
+            changed = _grubd_autopkgtest_setup(arch)
+        else:
+            changed = _grub_autopkgtest_setup(arch)
+        if changed:
+            subprocess.check_call(['update-grub'])
+
+
+def disable_qemu_setup_on_boot():
+    """Disable this setup script from running again on future boot up."""
+    subprocess.call(['systemctl', 'disable', 'qemu-setup.service'])
+
+
+def system_shutdown():
+    """Shutdown the system to complete first boot setup."""
+    subprocess.call(['shutdown', '--poweroff', '0'])
+
+
+def do_setup():
+    """Complete first boot setup actions."""
+    wait_for_network()
+    setup_serial_console()
+    install_packages(['openssh-server'])
+
+
+def do_teardown():
+    """Cleanup actions to disable this from running after first boot."""
+    disable_qemu_setup_on_boot()
+    system_shutdown()
+
+
+def main():
+    try:
+        do_setup()
+    finally:
+        do_teardown()
+
+
+if __name__ == '__main__':
+    main()

=== added file 'utah/provisioning/qemu-scripts/qemu-setup.service'
--- utah/provisioning/qemu-scripts/qemu-setup.service	1970-01-01 00:00:00 +0000
+++ utah/provisioning/qemu-scripts/qemu-setup.service	2017-07-06 18:08:08 +0000
@@ -0,0 +1,9 @@
+[Unit]
+Description=Ubuntu System Tests QEMU setup autostart script
+
+[Service]
+Type=oneshot
+ExecStart=/bin/sh -c "/usr/bin/python3 /usr/local/bin/qemu-setup.py > /dev/ttyS0 2>&1"
+
+[Install]
+WantedBy=multi-user.target

=== modified file 'utah/provisioning/vm.py'
--- utah/provisioning/vm.py	2016-12-23 15:01:36 +0000
+++ utah/provisioning/vm.py	2017-07-06 18:08:08 +0000
@@ -27,6 +27,7 @@
 
 from xml.etree import ElementTree
 
+from utah.cleanup import cleanup
 from utah.config import config
 from utah.process import ProcessChecker, ProcessRunner
 from utah.provisioning.exceptions import UTAHProvisioningException
@@ -518,6 +519,8 @@
             initrd_dir = os.path.join(tmpdir, 'initrd.d')
             provision_data.update_initrd(initrd_dir)
 
+        self._copy_qemu_scripts(tmpdir=tmpdir)
+
         self._setuplatecommand(tmpdir=tmpdir)
 
         self._setuppreseed(tmpdir=tmpdir)
@@ -546,6 +549,8 @@
         self.vm = self.lv.defineXML(ElementTree.tostring(xml.getroot()))
         self.cleanfunction(self.vm.destroy)
         self.cleanfunction(self.vm.undefine)
+        if self.poweroff:
+            self.cleanfunction(self.vm.shutdown, force=True)
 
     def _start(self):
         """Start the VM."""
@@ -620,6 +625,12 @@
             "UPDATE machines SET state='destroyed' ""WHERE machineid=?",
             [machineid])
 
+    def shut_off(self, machineid):
+        """Update the database to indicate a machine is shut off."""
+        self.execute(
+            "UPDATE machines SET state='shut off' ""WHERE machineid=?",
+            [machineid])
+
 
 def get_vm(**kw):
     """Return a Machine object for a VM with the passed in arguments.