Merge lp:~doanac/ubuntu-ci-services-itself/imgbuild-kernel-fixes into lp:ubuntu-ci-services-itself

[Andy Doan]

image-builder: fixes required to get kernel testing working

This fixes the issues I've discovered in the image-builder. The whole diff is small, but it might make more sense to read each commit on its own to see *why* I made these chages.

[Evan (ev)]

57 + _chrooted(chroot, 'apt-get install -y {}'.format(' '.join(pkglist)))

We have all the apt keys imported and the cache updated, yes? I'm typing this from my iPad, otherwise I'd check myself.

139 +image_size: 4G

Can we have a comment documenting what this does and why it's set to 4G?

Otherwise +1. Good stuff.

Have you checked with Paul on why we had those dpkg's options set in the first place?

review: Approve

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:403
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/438/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/438/rebuild

review: Approve (continuous-integration)

[Paul Larson (pwlars)]

> Have you checked with Paul on why we had those dpkg's options set in the first
> place?
Yes, those were set to get around grub wanting to give an interactive prompt to decide what to do about the config file. From what Andy and I discussed earlier on IRC, it should be ok to remove them since he has 'etc/default/grub.d/99-ci-eng.cfg' getting created and used, but I'm working on testing it now using this branch.

[lp:~doanac/ubuntu-ci-services-itself/imgbuild-kernel-fixes]

404. By Andy Doan on 2014-03-15

document the 4G size in our unit_config

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:404
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/439/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/439/rebuild

review: Approve (continuous-integration)

[Paul Larson (pwlars)]

I've been testing this over the weekend, and it's working well for me +1

review: Approve

=== modified file 'image-builder/imagebuilder/cloud_image.py'
--- image-builder/imagebuilder/cloud_image.py	2014-03-15 13:37:37 +0000
+++ image-builder/imagebuilder/cloud_image.py	2014-03-15 23:16:04 +0000
@@ -19,7 +19,6 @@
 import os
 import re
 import shutil
-import string
 import subprocess
 import textwrap
 import time
@@ -80,22 +79,30 @@
     log.info('Running command in image: {0}'.format(cmd))
     if type(cmd) is not list:
         cmd = cmd.split(' ')
-    output = subprocess.check_output(['chroot', chroot] + cmd)
+    output = subprocess.check_output(
+        ['chroot', chroot] + cmd, stderr=subprocess.STDOUT)
     if output:
         log.info(output)
 
 
 def _apt_get_update(chroot):
-    # apt-get update isn't always reliable give it 3 tries:
-    for x in (2, 1, 0):
+    # apt-get update isn't always reliable, give it a few tries
+    retries = 5
+    for x in range(retries):
         try:
             _chrooted(chroot, "/usr/bin/apt-get update")
             break
         except subprocess.CalledProcessError as e:
-            if x == 0:
+            x += 1
+            if x == retries:
                 raise e
             else:
-                log.info('apt-get update failed, retrying %d more times', x)
+                # retry delays will be 1, 16, 81, 256, 625
+                # ie about 1s, 16s, 1.5m, 4m, 10m
+                timeout = x * x * x * x
+                fmt = 'apt-get update failed, wait %ds and try %d more times'
+                log.info(fmt, timeout, retries - x)
+                time.sleep(timeout)
 
 
 def _setup_ppas(chroot, repos, series, status_cb):
@@ -124,11 +131,7 @@
     '''
     status_cb('Adding requested packages to the image...')
     os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
-    _chrooted(chroot,
-              ['/bin/bash', '-c',
-               ('/usr/bin/apt-get -o Dpkg::Options::=\"--force-confdef\" '
-                '-o Dpkg::Options::=\"--force-confold\" install -y '
-                '{pkglist}'.format(pkglist=string.join(pkglist, " ")))])
+    _chrooted(chroot, 'apt-get install -y {}'.format(' '.join(pkglist)))
     _chrooted(chroot, "/usr/bin/apt-get clean")
 
     #create a big null data file and remove it to make it more compressable
@@ -144,6 +147,19 @@
     os.unlink(zerofile)
 
 
+def _setup_grub(mountpoint, status_cb):
+    '''ensure grub uses the proper rootfs
+
+    When installing kernels grub will reconfigure but set the wrong root
+    device to boot from. This ensures its probed correctly
+    '''
+    status_cb('ensuring grub uses proper rootfs')
+    path = os.path.join(mountpoint, 'etc/default/grub.d/99-ci-eng.cfg')
+    with open(path, 'w') as f:
+        f.write('LINUX_ROOT_DEVICE="LABEL=cloudimg-rootfs"\n')
+        f.write('GRUB_DEVICE="LABEL=cloudimg-rootfs"\n')
+
+
 def _download(url, path):
     urlpath = urlparse.urlsplit(url).path
     filename = os.path.basename(urlpath)
@@ -166,6 +182,9 @@
 
 class CloudImage(object):
     def __init__(self, image_path, tmpdir):
+        image_size = ci_utils.unit_config.get('image_size')
+        log.info('resizing image to be %s', image_size)
+        subprocess.check_call(['qemu-img', 'resize', image_path, image_size])
         self.mountpoint = self._mount_image(image_path, tmpdir)
 
     def __enter__(self):
@@ -241,6 +260,8 @@
         with CloudImage(image_path, tmpdir) as mountpoint:
             status_cb('updating image %s...' % image)
 
+            _setup_grub(mountpoint, status_cb)
+
             _setup_ppas(mountpoint, repos, series, status_cb)
 
             _run_chroot_cmds(mountpoint, packages, status_cb)

=== modified file 'image-builder/imagebuilder/tests/test_modify_cloud_image.py'
--- image-builder/imagebuilder/tests/test_modify_cloud_image.py	2014-03-15 03:22:48 +0000
+++ image-builder/imagebuilder/tests/test_modify_cloud_image.py	2014-03-15 23:16:04 +0000
@@ -43,14 +43,16 @@
         expected = ['deb foo_url precise main', 'deb-src foo_url precise main']
         self.assertListEqual(expected, deblines)
 
+    @mock.patch('time.sleep')
     @mock.patch('imagebuilder.cloud_image._chrooted')
-    def test_apt_get_update_fails(self, chrooted):
+    def test_apt_get_update_fails(self, chrooted, sleep):
         chrooted.side_effect = subprocess.CalledProcessError(1, 'cmd', 'out')
         with self.assertRaises(subprocess.CalledProcessError):
             cloud_image._apt_get_update('/blah')
 
+    @mock.patch('time.sleep')
     @mock.patch('imagebuilder.cloud_image._chrooted')
-    def test_apt_get_update_retries(self, chrooted):
+    def test_apt_get_update_retries(self, chrooted, sleep):
         '''ensure it can apt-get update works after retrying'''
         chrooted.side_effect = [
             subprocess.CalledProcessError(1, 'cmd', 'out'),
@@ -59,6 +61,7 @@
         ]
         cloud_image._apt_get_update('/blah')
         self.assertEqual(3, chrooted.call_count)
+        self.assertEqual(2, sleep.call_count)
 
     @mock.patch('imagebuilder.cloud_image.launchpad')
     @mock.patch('imagebuilder.cloud_image._chrooted')

=== modified file 'juju-deployer/configs/unit_config.yaml.tmpl'
--- juju-deployer/configs/unit_config.yaml.tmpl	2014-03-14 23:53:04 +0000
+++ juju-deployer/configs/unit_config.yaml.tmpl	2014-03-15 23:16:04 +0000
@@ -21,6 +21,9 @@
 
 # for ticket system
 base_image: "http://cloud-images.ubuntu.com/releases/13.10/release-20131015/ubuntu-13.10-server-cloudimg-amd64-disk1.img"
+# Cloud images are 2G. We need more space for the packages that might
+# be added with tickets.
+image_size: 4G
 series: saucy
 master_ppa: $CI_MASTER_PPA
 tr_flavors: