Merge lp:~mwhudson/livecd-rootfs/live-server-kernel-futzing into lp:livecd-rootfs

[Michael Hudson-Doyle]

* Changes to kernel handling for live-server: do not include kernel/initrd
     in filesystem.squashfs but rather install it in a throwaway layer on top
     of installer.squashfs and fish kernel, initrd and modules out of that with
     an initrd hook that records kernel metapackage name in /run and mounts
     /lib/modules.

[lp:~mwhudson/livecd-rootfs/live-server-kernel-futzing]

1737. By Michael Hudson-Doyle on 2018-11-30

changelog

1738. By Michael Hudson-Doyle on 2018-11-30

fix syntax error

[lp:~mwhudson/livecd-rootfs/live-server-kernel-futzing]

1739. By Michael Hudson-Doyle on 2018-11-30

rationalize suffix handling

1740. By Michael Hudson-Doyle on 2018-11-30

we cannot put the modules squashfses directly in /casper

[Michael Hudson-Doyle (mwhudson)]

Pushed some small tweaks.

[Michael Hudson-Doyle (mwhudson)]

See also

https://code.launchpad.net/~mwhudson/debian-cd/kernel-bits/+merge/359884

and

https://code.launchpad.net/~mwhudson/ubuntu-cdimage/kernel-bits/+merge/359883

[lp:~mwhudson/livecd-rootfs/live-server-kernel-futzing]

1741. By Michael Hudson-Doyle on 2018-11-30

i was sure you could pass squashfses directly as lowerdir, apparently not

1742. By Michael Hudson-Doyle on 2018-12-02

testing vs reality

1743. By Michael Hudson-Doyle on 2018-12-03

remove cloud-initramfs-copymods and add comments

1744. By Michael Hudson-Doyle on 2018-12-03

remove cloud-initramfs-copymods harder

1745. By Michael Hudson-Doyle on 2018-12-03

ffs

[Steve Langasek (vorlon)]

LGTM

review: Approve

[lp:~mwhudson/livecd-rootfs/live-server-kernel-futzing]

1746. By Michael Hudson-Doyle on 2018-12-03

debugging by guesswork: no squashfuse

1747. By Michael Hudson-Doyle on 2018-12-04

ffs

1748. By Michael Hudson-Doyle on 2018-12-04

ffs^2

1749. By Michael Hudson-Doyle on 2018-12-05

stop trying to be cute

1750. By Michael Hudson-Doyle on 2018-12-06

i h8te shell

[Michael Hudson-Doyle (mwhudson)]

This works now. New related merge proposals that must be merged first:

https://code.launchpad.net/~mwhudson/debian-cd/kernel-bits/+merge/360172

https://code.launchpad.net/~mwhudson/ubuntu-cdimage/kernel-bits-2/+merge/360039

[lp:~mwhudson/livecd-rootfs/live-server-kernel-futzing]

1751. By Michael Hudson-Doyle on 2018-12-09

consistently use tabs for indentation

1752. By Michael Hudson-Doyle on 2018-12-09

remove a comment that no longer makes sense

[Balint Reczey (rbalint)]

The target Bazaar branch is not active anymore.
Please resubmit the merge proposal against https://code.launchpad.net/~ubuntu-core-dev/livecd-rootfs/+git/livecd-rootfs/+ref/ubuntu/master .

[Steve Langasek (vorlon)]

this all landed in git.

=== modified file 'debian/changelog'
--- debian/changelog	2018-12-06 11:38:12 +0000
+++ debian/changelog	2018-12-09 23:08:48 +0000
@@ -1,3 +1,4 @@
+<<<<<<< TREE
 livecd-rootfs (2.548) disco; urgency=medium
 
   * Key netplan delegation to NetworkManager on presence of
@@ -6,6 +7,18 @@
 
  -- Steve Langasek <[e-mail]>  Thu, 06 Dec 2018 13:38:09 +0200
 
+=======
+livecd-rootfs (2.548~ppa13) disco; urgency=medium
+
+  * Changes to kernel handling for live-server: do not include kernel/initrd
+    in filesystem.squashfs but rather install it in a throwaway layer on top
+    of installer.squashfs and fish kernel, initrd and modules out of that with
+    an initrd hook that records kernel metapackage name in /run and mounts
+    /lib/modules from a squashfs on the ISO.
+
+ -- Michael Hudson-Doyle <[e-mail]>  Fri, 30 Nov 2018 14:13:20 +1300
+
+>>>>>>> MERGE-SOURCE
 livecd-rootfs (2.547) disco; urgency=medium
 
   [ Łukasz Zemczak ]

=== modified file 'live-build/auto/config'
--- live-build/auto/config	2018-11-28 15:05:24 +0000
+++ live-build/auto/config	2018-12-09 23:08:48 +0000
@@ -117,12 +117,6 @@
 	esac
 fi
 
-skip_lb_stage() {
-	STAGE="$1"
-	mkdir -p .build
-	touch ".build/$STAGE"
-}
-
 case $IMAGEFORMAT in
 	ext2|ext3|ext4)
 		OPTS="${OPTS:+$OPTS }--initramfs none --chroot-filesystem $IMAGEFORMAT"
@@ -145,20 +139,15 @@
 		;;
 
 	plain)
-		INITRAMFS_TYPE=none
 		case $PROJECT:${SUBPROJECT:-} in
 			ubuntu-server:live)
-				# Stop lb installing casper into filesystem.squashfs
-				#  by skipping lb_chroot_live-packages.
-				skip_lb_stage chroot_live-packages
-				INITRAMFS_TYPE=auto
 	                        touch config/universe-enabled
 				;;
 			*)
 				PREINSTALLED=true
 				;;
 		esac
-		OPTS="${OPTS:+$OPTS }--initramfs $INITRAMFS_TYPE --chroot-filesystem $IMAGEFORMAT"
+		OPTS="${OPTS:+$OPTS }--initramfs none --chroot-filesystem $IMAGEFORMAT"
 		;;
 
 	ubuntu-image)
@@ -519,9 +508,6 @@
 				add_task install standard
 				add_task install server
 				LIVE_TASK='cloud-image'
-				case $ARCH in
-					amd64)	add_package live linux-signed-generic ;;
-				esac
 				;;
 		esac
 		COMPONENTS='main'
@@ -837,8 +823,6 @@
 esac
 
 case $PROJECT:${SUBPROJECT:-} in
-	ubuntu-server:live)
-		;;
 	ubuntu-server:*|ubuntu-base:*|ubuntu-touch:*|ubuntu-touch-custom:*)
 		OPTS="${OPTS:+$OPTS }--linux-packages=none --initramfs=none"
 		KERNEL_FLAVOURS=none

=== modified file 'live-build/functions'
--- live-build/functions	2018-11-13 19:21:19 +0000
+++ live-build/functions	2018-12-09 23:08:48 +0000
@@ -337,37 +337,6 @@
 		--rename /usr/bin/systemd-detect-virt
 }
 
-recreate_initramfs() {
-	# Regenerate the initramfs by running update-initramfs in the
-	# chroot at $1 and copying the generated initramfs
-	# around. Beware that this was written for a single use case
-	# (live-server) and may not work in all cases without
-	# tweaking...
-	# config/common must be sourced before calling this function.
-	CHROOT="$1"
-	# Start by cargo culting bits of lb_chroot_hacks:
-	if [ -n "$LB_INITRAMFS_COMPRESSION" ]; then
-		echo "COMPRESS=$LB_INITRAMFS_COMPRESSION" > "$CHROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf
-	fi
-	chroot "$CHROOT" sh -c "${UPDATE_INITRAMFS_OPTIONS:-} update-initramfs -k all -t -u"
-	rm -rf "$CHROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf
-	# Then bits of lb_binary_linux-image:
-	case "${LB_INITRAMFS}" in
-		casper)
-			DESTDIR="binary/casper"
-			;;
-
-		live-boot)
-			DESTDIR="binary/live"
-			;;
-
-		*)
-			DESTDIR="binary/boot"
-			;;
-	esac
-	mv "$CHROOT"/boot/initrd.img-* $DESTDIR
-}
-
 release_ver() {
     # Return the release version number
     distro-info --series="$LB_DISTRIBUTION" -r | awk '{ print $1 }'

=== modified file 'live-build/ubuntu-server/hooks/032-installer-squashfs.binary'
--- live-build/ubuntu-server/hooks/032-installer-squashfs.binary	2018-11-12 20:39:25 +0000
+++ live-build/ubuntu-server/hooks/032-installer-squashfs.binary	2018-12-09 23:08:48 +0000
@@ -24,13 +24,14 @@
 . config/functions
 . config/common
 
-SQUASH_ROOT=binary/boot/squashfs.dir
+FILESYSTEM_ROOT=binary/boot/squashfs.dir
+INSTALLER_ROOT=binary/boot/installer.squashfs.dir
 OVERLAY_ROOT=binary/overlay
 
-mkdir -p "$OVERLAY_ROOT"
+mkdir -p "$INSTALLER_ROOT" "$OVERLAY_ROOT"
 
 # Create an installer squashfs layer
-mount_overlay "$SQUASH_ROOT/" "$OVERLAY_ROOT/" "$SQUASH_ROOT/"
+mount_overlay "$FILESYSTEM_ROOT/" "$OVERLAY_ROOT/" "$INSTALLER_ROOT/"
 
 setup_mountpoint binary/boot/squashfs.dir
 
@@ -40,8 +41,8 @@
 # It would be better to have this in ../includes.binary/overlay but
 # you can't have backslashes in filenames in bzr branches!
 DEVICE_UNIT_NAME='dev-disk-by\x2duuid-00c629d6\x2d06ab\x2d4dfd\x2db21e\x2dc3186f34105d.device'
-mkdir -p "$SQUASH_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d"
-cat > "$SQUASH_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d/override.conf" <<EOF
+mkdir -p "$INSTALLER_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d"
+cat > "$INSTALLER_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d/override.conf" <<EOF
 [Unit]
 JobRunningTimeoutSec=0s
 EOF
@@ -50,22 +51,19 @@
 
 # Install any requirements for the installer, for things we don't want
 # to see on the installed system
-chroot $SQUASH_ROOT apt-get update
-chroot $SQUASH_ROOT apt-get -y install user-setup curtin lupin-casper
+chroot $INSTALLER_ROOT apt-get update
+chroot $INSTALLER_ROOT apt-get -y install user-setup curtin lupin-casper
 
 # For bug #1743643 "Install to dirty disk with swap fails" remove the
 # "helpful" casper script that mounts any swap partitions it finds.
-rm -f $SQUASH_ROOT/usr/share/initramfs-tools/scripts/casper-bottom/*swap
-
-# Installing casper means we need a new initramfs
-UPDATE_INITRAMFS_OPTIONS=CASPER_GENERATE_UUID=1 recreate_initramfs $SQUASH_ROOT
+rm -f $INSTALLER_ROOT/usr/share/initramfs-tools/scripts/casper-bottom/*swap
 
 # Don't let cloud-init run in the live session.
-touch $SQUASH_ROOT/etc/cloud/cloud-init.disabled
+touch $INSTALLER_ROOT/etc/cloud/cloud-init.disabled
 
 # Do the snap seeding dance.
-chroot $SQUASH_ROOT mkdir -p /var/lib/snapd/seed/snaps /var/lib/snapd/seed/assertions
-chroot $SQUASH_ROOT sh -c '
+chroot $INSTALLER_ROOT mkdir -p /var/lib/snapd/seed/snaps /var/lib/snapd/seed/assertions
+chroot $INSTALLER_ROOT sh -c '
 set -x;
 cd /var/lib/snapd/seed;
 sudo SNAPPY_STORE_NO_CDN=1 snap download core;
@@ -89,10 +87,7 @@
 EOF
 '
 
-teardown_mountpoint "$SQUASH_ROOT"
-
-# Then unmount the overlay
-umount "$SQUASH_ROOT"
+teardown_mountpoint "$INSTALLER_ROOT"
 
 squashfs_f="${PWD}/livecd.${PROJECT}.installer.squashfs"
 

=== added file 'live-build/ubuntu-server/hooks/033-kernel-bits.binary'
--- live-build/ubuntu-server/hooks/033-kernel-bits.binary	1970-01-01 00:00:00 +0000
+++ live-build/ubuntu-server/hooks/033-kernel-bits.binary	2018-12-09 23:08:48 +0000
@@ -0,0 +1,94 @@
+#!/bin/bash -eux
+# vi: ts=4 noexpandtab
+#
+# Generate a squashfs root and manifest
+
+echo "033-kernel-bits.binary"
+
+case ${IMAGE_TARGETS-} in
+	""|*squashfs*)
+		;;
+	*)
+		echo "Skipping squashfs build"
+		exit 0
+		;;
+esac
+
+if [ -n "${SUBARCH-}" ]; then
+	echo "Skipping rootfs build for subarch flavor build"
+	exit 0
+fi
+
+. config/functions
+. config/common
+
+INSTALLER_ROOT=binary/boot/installer.squashfs.dir
+
+KERNEL_BITS_ROOT=binary/boot/kernel-bits.dir
+KERNEL_BITS_OVERLAY=binary/boot/overlay-kernel-bits
+
+#variants='ga hwe'
+variants='ga'
+
+for variant in $variants; do
+	if [ "$variant" = "ga" ]; then
+		kernel_metapkg=linux-generic
+		modules_squashfs_f=modules.squashfs
+		initrd_f=initrd-generic
+		kernel_f=kernel-generic
+	elif [ "$variant" = "hwe" ]; then
+		kernel_metapkg=linux-generic-hwe-$(lsb_release -sc)
+		modules_squashfs_f=modules-hwe.squashfs
+		initrd_f=initrd-generic-hwe
+		kernel_f=kernel-generic-hwe
+	else
+		echo "bogus variant: $variant"
+		exit 1
+	fi
+
+	# Make preparations
+	mkdir -p $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY
+	mount_overlay "$INSTALLER_ROOT/" "$KERNEL_BITS_OVERLAY/" "$KERNEL_BITS_ROOT/"
+	setup_mountpoint $KERNEL_BITS_ROOT
+
+	# Our initramfs hook implements a kind of extreme version of
+	# cloud-initramfs-copymods, so remove that and prevent duelling hooks
+	env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y remove cloud-initramfs-copymods ubuntu-server || true
+
+	# Configure initramfs creation
+	if [ -n "$LB_INITRAMFS_COMPRESSION" ]; then
+		echo "COMPRESS=$LB_INITRAMFS_COMPRESSION" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf
+	fi
+	echo "CASPER_GENERATE_UUID=1" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/casper.conf
+
+	# Add a hook to record which kernel was booted and mount the
+	# modules.squashfs created below.
+	cat <<EOF > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server
+#!/bin/sh
+case \$1 in
+prereqs) exit 0;;
+esac
+
+echo ${kernel_metapkg} > /run/kernel-meta-package
+mkdir -p \$rootmnt/lib/modules
+mount \$rootmnt/cdrom/casper/extras/$modules_squashfs_f \$rootmnt/lib/modules
+EOF
+	chmod +x "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server
+
+	# Install the kernel!
+	env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y install ${kernel_metapkg}
+
+	# Fish out generated kernel image and initrd
+	mv "$KERNEL_BITS_ROOT"/boot/initrd.img-* ${PWD}/livecd.${PROJECT}.$initrd_f
+	mv "$KERNEL_BITS_ROOT"/boot/vmlinu?-* ${PWD}/livecd.${PROJECT}.$kernel_f
+
+	# Create squashfs containing all the modules
+	modules_squashfs_path="${PWD}/livecd.${PROJECT}.$modules_squashfs_f"
+	(cd "$KERNEL_BITS_ROOT/lib/modules" &&
+		mksquashfs . $modules_squashfs_path -no-progress -xattrs -comp xz)
+
+	# And clean up
+	teardown_mountpoint $KERNEL_BITS_ROOT
+	umount $KERNEL_BITS_ROOT
+	rm -rf $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY
+done