Merge lp:~mwhudson/ubuntu-cdimage/kernel-bits-2 into lp:ubuntu-cdimage

[Michael Hudson-Doyle]

download hwe boot bits for bionic live server builds

Unlike my other branch, this one should work.

[Adam Conrad (adconrad)]

Some thoughts in inline comments.

[lp:~mwhudson/ubuntu-cdimage/kernel-bits-2]

1766. By Michael Hudson-Doyle on 2018-12-05

simplifications per review

1767. By Michael Hudson-Doyle on 2018-12-06

need this

[Michael Hudson-Doyle (mwhudson)]

I've addressed Adam's comments (and updated my livecd-rootfs branch to match)

[Steve Langasek (vorlon)]

review: Needs Fixing

[lp:~mwhudson/ubuntu-cdimage/kernel-bits-2]

1768. By Michael Hudson-Doyle on 2018-12-12

make names more consistent

=== modified file 'lib/cdimage/livefs.py'
--- lib/cdimage/livefs.py	2018-03-02 15:28:21 +0000
+++ lib/cdimage/livefs.py	2018-12-12 20:29:45 +0000
@@ -676,9 +676,11 @@
             for url in urls_for("livecd.%s.%s" % (liveproject_subarch, item)):
                 yield url
     elif item in (
-        "kernel", "initrd", "bootimg"
+        "kernel", "initrd", "bootimg", "modules.squashfs"
     ):
-        for flavour in flavours(config, arch):
+        our_flavours = flavours(config, arch)
+        our_flavours.extend(["%s-hwe" % (f,) for f in our_flavours])
+        for flavour in our_flavours:
             base = "livecd.%s.%s-%s" % (liveproject_subarch, item, flavour)
             for url in urls_for(base):
                 yield url
@@ -776,6 +778,18 @@
             except osextras.FetchError:
                 pass
     elif item in (
+        "modules.squashfs",
+    ):
+        for url in urls:
+            base = unquote(os.path.basename(url))
+            base = "%s.%s" % (arch, base.split('.', 2)[2])
+            target = os.path.join(output_dir, base)
+            try:
+                osextras.fetch(config, url, target)
+                found = True
+            except osextras.FetchError:
+                pass
+    elif item in (
         "recovery-%s+%s.img" % (target.android_arch, target.subarch)
             for target in Touch.list_targets_by_ubuntu_arch(arch)
     ):
@@ -898,6 +912,7 @@
                 download_live_items(config, arch, "installer.squashfs")
                 download_live_items(config, arch, "maas-rack.squashfs")
                 download_live_items(config, arch, "maas-region.squashfs")
+                download_live_items(config, arch, "modules.squashfs")
                 got_image = True
             elif download_live_items(config, arch, "rootfs.tar.gz"):
                 got_image = True

=== modified file 'lib/cdimage/tests/test_livefs.py'
--- lib/cdimage/tests/test_livefs.py	2018-03-02 14:21:47 +0000
+++ lib/cdimage/tests/test_livefs.py	2018-12-12 20:29:45 +0000
@@ -1032,13 +1032,16 @@
         for item in ("kernel", "initrd", "bootimg"):
             root = "http://kapok.buildd/~buildd/LiveCD/precise/kubuntu/current"
             self.assertPathsEqual(
-                ["%s/livecd.kubuntu.%s-generic" % (root, item)],
+                ["%s/livecd.kubuntu.%s-generic" % (root, item),
+                 "%s/livecd.kubuntu.%s-generic-hwe" % (root, item)],
                 "amd64", item, "kubuntu", "precise")
             root = ("http://royal.buildd/~buildd/LiveCD/hardy/ubuntu-ps3/"
                     "current")
             self.assertPathsEqual(
                 ["%s/livecd.ubuntu-ps3.%s-powerpc" % (root, item),
-                 "%s/livecd.ubuntu-ps3.%s-powerpc64-smp" % (root, item)],
+                 "%s/livecd.ubuntu-ps3.%s-powerpc64-smp" % (root, item),
+                 "%s/livecd.ubuntu-ps3.%s-powerpc-hwe" % (root, item),
+                 "%s/livecd.ubuntu-ps3.%s-powerpc64-smp-hwe" % (root, item)],
                 "powerpc+ps3", item, "ubuntu", "hardy")
 
     def test_kernel_efi_signed(self):
@@ -1206,11 +1209,16 @@
         self.assertTrue(download_live_items(self.config, "i386", "kernel"))
         prefix = ("http://cardamom.buildd/~buildd/LiveCD/raring/ubuntu/"
                   "current/livecd.ubuntu.kernel-")
-        target_dir = os.path.join(
-            self.temp_dir, "scratch", "ubuntu", "raring", "daily-live", "live")
-        mock_fetch.assert_called_once_with(
-            self.config, prefix + "generic",
-            os.path.join(target_dir, "i386.kernel-generic"))
+        calls = []
+        for suffix in '', '-hwe':
+            target_dir = os.path.join(
+                self.temp_dir, "scratch", "ubuntu", "raring", "daily-live",
+                "live")
+            calls.append(
+                mock.call(
+                    self.config, prefix + "generic" + suffix,
+                    os.path.join(target_dir, "i386.kernel-generic" + suffix)))
+        self.assertCountEqual(mock_fetch.call_args_list, calls)
 
     @mock.patch("cdimage.osextras.fetch")
     def test_download_live_items_kernel_efi_signed(self, mock_fetch):
@@ -1236,12 +1244,17 @@
             download_live_items(self.config, "armhf+omap4", "bootimg"))
         url = ("http://kishi00.buildd/~buildd/LiveCD/raring/ubuntu-omap4/"
                "current/livecd.ubuntu-omap4.bootimg-omap4")
-        target_dir = os.path.join(
-            self.temp_dir, "scratch", "ubuntu", "raring", "daily-preinstalled",
-            "live")
-        mock_fetch.assert_called_once_with(
-            self.config, url,
-            os.path.join(target_dir, "armhf+omap4.bootimg-omap4"))
+        calls = []
+        for suffix in '', '-hwe':
+            target_dir = os.path.join(
+                self.temp_dir, "scratch", "ubuntu", "raring",
+                "daily-preinstalled", "live")
+            calls.append(
+                mock.call(
+                    self.config, url + suffix,
+                    os.path.join(
+                        target_dir, "armhf+omap4.bootimg-omap4" + suffix)))
+        self.assertCountEqual(mock_fetch.call_args_list, calls)
 
     @mock.patch("cdimage.osextras.fetch")
     def test_download_live_items_wubi(self, mock_fetch):
@@ -1321,55 +1334,46 @@
             self.config, url, os.path.join(target_dir, "i386.server-squashfs"))
 
     @mock.patch("cdimage.osextras.fetch")
-    def test_download_live_items_maas_rack_squashfs(self, mock_fetch):
-        self.config["PROJECT"] = "ubuntu-server"
-        self.config["SUBPROJECT"] = "live"
-        self.config["DIST"] = "bionic"
-        self.config["IMAGE_TYPE"] = "daily-live"
-        self.assertTrue(
-            download_live_items(self.config, "amd64", "maas-rack.squashfs"))
-        url = ("http://kapok.buildd/~buildd/LiveCD/bionic/ubuntu-server-live/"
-               "current/livecd.ubuntu-server.maas-rack.squashfs")
-        target_dir = os.path.join(
-            self.temp_dir, "scratch", "ubuntu-server", "bionic", "daily-live",
-            "live")
-        mock_fetch.assert_called_once_with(
-            self.config, url, os.path.join(target_dir,
-                                           "amd64.maas-rack.squashfs"))
-
-    @mock.patch("cdimage.osextras.fetch")
-    def test_download_live_items_maas_region_squashfs(self, mock_fetch):
-        self.config["PROJECT"] = "ubuntu-server"
-        self.config["SUBPROJECT"] = "live"
-        self.config["DIST"] = "bionic"
-        self.config["IMAGE_TYPE"] = "daily-live"
-        self.assertTrue(
-            download_live_items(self.config, "amd64", "maas-region.squashfs"))
-        url = ("http://kapok.buildd/~buildd/LiveCD/bionic/ubuntu-server-live/"
-               "current/livecd.ubuntu-server.maas-region.squashfs")
-        target_dir = os.path.join(
-            self.temp_dir, "scratch", "ubuntu-server", "bionic", "daily-live",
-            "live")
-        mock_fetch.assert_called_once_with(
-            self.config, url, os.path.join(target_dir,
-                                           "amd64.maas-region.squashfs"))
-
-    @mock.patch("cdimage.osextras.fetch")
-    def test_download_live_items_installer_squashfs(self, mock_fetch):
-        self.config["PROJECT"] = "ubuntu-server"
-        self.config["SUBPROJECT"] = "live"
-        self.config["DIST"] = "xenial"
-        self.config["IMAGE_TYPE"] = "daily-live"
-        self.assertTrue(
-            download_live_items(self.config, "amd64", "installer.squashfs"))
-        url = ("http://kapok.buildd/~buildd/LiveCD/xenial/ubuntu-server-live/"
-               "current/livecd.ubuntu-server.installer.squashfs")
-        target_dir = os.path.join(
-            self.temp_dir, "scratch", "ubuntu-server", "xenial", "daily-live",
-            "live")
-        mock_fetch.assert_called_once_with(
-            self.config, url, os.path.join(target_dir,
-                                           "amd64.installer.squashfs"))
+    def assert_server_live_download_items(self, series, item, filenames,
+                                          mock_fetch):
+        self.config["PROJECT"] = "ubuntu-server"
+        self.config["SUBPROJECT"] = "live"
+        self.config["DIST"] = series
+        self.config["IMAGE_TYPE"] = "daily-live"
+        self.assertTrue(
+            download_live_items(self.config, "amd64", item))
+        calls = []
+        for filename in filenames:
+            url = ("http://kapok.buildd/~buildd/LiveCD/%s/ubuntu-server-live/"
+                   "current/livecd.ubuntu-server.%s") % (series, filename)
+            target = os.path.join(
+                self.temp_dir, "scratch", "ubuntu-server", series,
+                "daily-live", "live", "amd64." + filename)
+            calls.append(mock.call(self.config, url, target))
+        self.assertCountEqual(mock_fetch.call_args_list, calls)
+
+    def test_download_live_items_installer_squashfs(self):
+        self.assert_server_live_download_items(
+            "bionic", "installer.squashfs", ["installer.squashfs"])
+
+    def test_download_live_items_maas_rack_squashfs(self):
+        self.assert_server_live_download_items(
+            "bionic", "maas-rack.squashfs", ["maas-rack.squashfs"])
+
+    def test_download_live_items_maas_region_squashfs(self):
+        self.assert_server_live_download_items(
+            "bionic", "maas-region.squashfs", ["maas-region.squashfs"])
+
+    def test_download_live_server_boot_items(self):
+        self.assert_server_live_download_items(
+            "bionic", "kernel",
+            ["kernel-generic", "kernel-generic-hwe"])
+        self.assert_server_live_download_items(
+            "bionic", "initrd",
+            ["initrd-generic", "initrd-generic-hwe"])
+        self.assert_server_live_download_items(
+            "bionic", "modules.squashfs",
+            ["modules.squashfs-generic", "modules.squashfs-generic-hwe"])
 
     def test_write_autorun(self):
         self.config["PROJECT"] = "ubuntu"