Merge lp:~sil2100/livecd-rootfs/bionic-raspi3-backport into lp:~ubuntu-core-dev/livecd-rootfs/bionic-proposed

[Łukasz Zemczak]

Backport all the necessary livecd-rootfs changes for raspi3 armhf/arm64 support.

Backport all the necessary livecd-rootfs changes for raspi3 armhf/arm64 support.

[Łukasz Zemczak (sil2100)]

I tested it in a PPA and performed a livefs build of raspi3+arm64 - resulting image works as expected. Apologies for the self-approval but we're short on time..!

review: Approve

[Łukasz Zemczak (sil2100)]

https://launchpad.net/~canonical-foundations/+livefs/ubuntu/bionic/ubuntu-raspi/+build/151105 (livefs build link)

=== modified file 'debian/changelog'
--- debian/changelog	2018-11-26 20:55:29 +0000
+++ debian/changelog	2018-11-29 16:44:19 +0000
@@ -1,3 +1,20 @@
+livecd-rootfs (2.525.11) UNRELEASED; urgency=medium
+
+  * Backport all the required changes to enable Raspberry Pi 3 armhf and arm64
+    preinstalled image builds. (LP: #1805668)
+    - Add support for raspi3 rootfs builds (based on Ryan Finnie's changes).
+    - For ubuntu-image consumption, export the kernel and initrd to
+      image/boot/uboot for raspi*.
+    - Avoid issues of hard-linking to a symbolic vmlinuz as this can lead to a
+      dangling symlink.
+    - Add raspi3 arm64 rootfs build support.
+    - Add git to the build dependencies for the gadget tree pull.
+    - Minor fixes to raspi3 builds: add arm64+raspi3 to the supported model
+      list, pass SUITE on to the build stage and use the git:// url for the
+      gadget tree.
+
+ -- Łukasz 'sil2100' Zemczak <[e-mail]>  Thu, 29 Nov 2018 16:24:23 +0100
+
 livecd-rootfs (2.525.10) bionic; urgency=medium
 
   [ Cody Shepherd ]

=== modified file 'debian/control'
--- debian/control	2018-10-26 21:01:17 +0000
+++ debian/control	2018-11-29 16:44:19 +0000
@@ -17,12 +17,14 @@
          gdisk,
          genisoimage,
          germinate (>= 1.25.1),
+         git,
          gnupg,
          grep-dctrl,
          kpartx,
          live-build (>= 3.0~a57-1ubuntu31~),
          lsb-release,
          lzma,
+         make,
          parted,
          procps,
          python-minimal | python,
@@ -32,6 +34,7 @@
          snapd,
          squashfs-tools (>= 1:3.3-1),
          sudo,
+         u-boot-tools [armhf arm64],
          ubuntu-image,
          vmdk-stream-converter [amd64 i386],
          xz-utils,

=== modified file 'live-build/auto/build'
--- live-build/auto/build	2018-08-27 22:32:07 +0000
+++ live-build/auto/build	2018-11-29 16:44:19 +0000
@@ -23,12 +23,31 @@
 if [ "${IMAGEFORMAT:-}" = "ubuntu-image" ]; then
 	# Use ubuntu-image instead of live-build
 
-	CHANNEL="${CHANNEL:-edge}"
-	env SNAPPY_STORE_NO_CDN=1 \
-	ubuntu-image -c "$CHANNEL" $UBUNTU_IMAGE_ARGS \
-		-o "$PREFIX".img "$PREFIX".model-assertion
-	xz -0 -T4 "$PREFIX".img
-	mv seed.manifest "$PREFIX".manifest
+	if [ "$PROJECT" = "ubuntu-core"]; then
+		CHANNEL="${CHANNEL:-edge}"
+		env SNAPPY_STORE_NO_CDN=1 \
+		ubuntu-image snap -c "$CHANNEL" $UBUNTU_IMAGE_ARGS \
+			-O output "$PREFIX".model-assertion
+		# XXX: currently we only have one image generated, but really
+		#  we should be supporting more than one for models that
+		#  define those.
+		mv output/*.img "$PREFIX".img
+		xz -0 -T4 "$PREFIX".img
+		mv output/seed.manifest "$PREFIX".manifest
+	else
+		# First we need to build the gadget tree
+		make -C "config/$PREFIX-gadget" ARCH=$ARCH SERIES=$SUITE
+		ubuntu-image classic $UBUNTU_IMAGE_ARGS \
+			-s $SUITE -p $PROJECT -a $ARCH --subarch $SUBARCH \
+			-O output config/$PREFIX-gadget/install
+		# XXX: currently we only have one image generated, but really
+		#  we should be supporting more than one for models that
+		#  define those.
+		mv output/*.img "$PREFIX".img
+		xz -0 -T4 "$PREFIX".img
+		mv output/filesystem.manifest "$PREFIX".manifest
+	fi
+
 	exit 0
 fi
 
@@ -800,13 +819,13 @@
 NUMFLAVOURS="$(set -- $LB_LINUX_FLAVOURS; echo $#)"
 if [ "$NUMFLAVOURS" = 1 ] && [ "$LB_LINUX_FLAVOURS" != "none" ]; then
 	# only one kernel flavour
-	if [ -e "binary/$INITFS/vmlinuz" ]; then
+	if [ -f "binary/$INITFS/vmlinuz" ] && ! [ -h "binary/$INITFS/vmlinuz" ]; then
 		ln "binary/$INITFS/vmlinuz" "$PREFIX.kernel"
 		chmod 644 "$PREFIX.kernel"
 	else
 		ln -sf "$PREFIX.kernel-$LB_LINUX_FLAVOURS" "$PREFIX.kernel"
 	fi
-	if [ -e "binary/$INITFS/initrd.lz" ]; then
+	if [ -f "binary/$INITFS/initrd.lz" ] && ! [ -h "binary/$INITFS/initrd.lz" ]; then
 		ln "binary/$INITFS/initrd.lz" "$PREFIX.initrd"
 		chmod 644 "$PREFIX.initrd"
 	else
@@ -814,36 +833,52 @@
 	fi
 fi
 
-if [ "$SUBARCH" = "ac100" ]; then
-	# create the md5sum and size files for which we are actually doing all this
-	md5sum $PREFIX.rootfs.tar.gz >chroot/installer.md5
-	wc -c $PREFIX.rootfs.tar.gz >chroot/installer.size
-
-	INFO_DESC="$(lsb_release -d -s)"
-	INFO_STAMP=$(date +20%y%m%d-%H:%M)
-
-	echo "$INFO_DESC - $ARCH ($INFO_STAMP)" >chroot/media-info
-
-	# make sure update-initramfs feels cosy and warm in the environment
-	lb chroot_proc install "$@"
-	lb chroot_sysfs install "$@"
-	lb chroot_devpts install "$@"
-
-	# re-create initrd to contain the installer.md5 file
-	Chroot chroot "env FLASH_KERNEL_SKIP=1 update-initramfs -k all -t -u -v"
-
-	# create boot.img
-	Chroot chroot "abootimg --create /boot/installer-${KVERS}.img -f /boot/bootimg.cfg-$SUBARCH -r /boot/initrd.img-${KVERS} -k /boot/vmlinuz-${KVERS}"
-
-	# clean up
-	lb chroot_devpts remove "$@"
-	lb chroot_sysfs remove "$@"
-	lb chroot_proc remove "$@"
-
-	cp "chroot/boot/installer-${KVERS}.img" "$PREFIX.bootimg-$FLAVOUR"
-	ln -sf "$PREFIX.bootimg-$FLAVOUR" "$PREFIX.bootimg"
-
-fi
+case $SUBARCH in
+	ac100)
+		# create the md5sum and size files for which we are actually doing all this
+		md5sum $PREFIX.rootfs.tar.gz >chroot/installer.md5
+		wc -c $PREFIX.rootfs.tar.gz >chroot/installer.size
+	
+		INFO_DESC="$(lsb_release -d -s)"
+		INFO_STAMP=$(date +20%y%m%d-%H:%M)
+	
+		echo "$INFO_DESC - $ARCH ($INFO_STAMP)" >chroot/media-info
+	
+		# make sure update-initramfs feels cosy and warm in the environment
+		lb chroot_proc install "$@"
+		lb chroot_sysfs install "$@"
+		lb chroot_devpts install "$@"
+	
+		# re-create initrd to contain the installer.md5 file
+		Chroot chroot "env FLASH_KERNEL_SKIP=1 update-initramfs -k all -t -u -v"
+	
+		# create boot.img
+		Chroot chroot "abootimg --create /boot/installer-${KVERS}.img -f /boot/bootimg.cfg-$SUBARCH -r /boot/initrd.img-${KVERS} -k /boot/vmlinuz-${KVERS}"
+	
+		# clean up
+		lb chroot_devpts remove "$@"
+		lb chroot_sysfs remove "$@"
+		lb chroot_proc remove "$@"
+	
+		cp "chroot/boot/installer-${KVERS}.img" "$PREFIX.bootimg-$FLAVOUR"
+		ln -sf "$PREFIX.bootimg-$FLAVOUR" "$PREFIX.bootimg"
+		;;
+
+	raspi2|raspi3)
+		# copy the kernel and initrd to a predictable directory for
+		# ubuntu-image consumption.  In some cases, like in pi2/3
+		# u-boot, the bootloader needs to contain the kernel and initrd,
+		# so during rootfs build we copy it over to a directory that
+		# ubuntu-image looks for and shoves into the bootloader
+		# partition.
+		UBOOT_BOOT="image/boot/uboot"
+
+		mkdir -p $UBOOT_BOOT
+
+		cp $PREFIX.initrd $UBOOT_BOOT/initrd.img || true
+		cp $PREFIX.kernel $UBOOT_BOOT/vmlinuz || true
+		;;
+esac
 
 if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then
 	sourceslist="chroot/etc/apt/sources.list"

=== modified file 'live-build/auto/config'
--- live-build/auto/config	2018-10-24 19:18:36 +0000
+++ live-build/auto/config	2018-11-29 16:44:19 +0000
@@ -175,7 +175,7 @@
 				MODEL=dragonboard ;;
 			armhf+raspi2)
 				MODEL=pi2 ;;
-			armhf+raspi3)
+			armhf+raspi3|arm64+raspi3)
 				MODEL=pi3 ;;
 			armhf+cm3)
 				MODEL=cm3 ;;
@@ -184,20 +184,39 @@
 				exit 1
 				;;
 		esac
-		case $MODEL in
-			pc-amd64|pc-i386)
-				UBUNTU_IMAGE_ARGS="--image-size 3700M" ;;
-			*)
-				UBUNTU_IMAGE_ARGS="" ;;
-		esac
-
-		echo "IMAGEFORMAT=$IMAGEFORMAT" >> config/common
-		echo "UBUNTU_IMAGE_ARGS=\"$UBUNTU_IMAGE_ARGS\"" >> config/common
-		# Store model assertion in top dir to get it picked up later as a build artifact
-		env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 model="$MODEL" brand-id=canonical > "$PREFIX".model-assertion
-		echo "Configured ubuntu-image for the following model assertion:"
-		cat "$PREFIX".model-assertion
-		echo "----------------------------------------------------------"
+
+		if [ $PROJECT = "ubuntu-core" ]; then
+			# snap-based core images
+
+			case $MODEL in
+				pc-amd64|pc-i386)
+					UBUNTU_IMAGE_ARGS="--image-size 3700M" ;;
+				*)
+					UBUNTU_IMAGE_ARGS="" ;;
+			esac
+	
+			echo "IMAGEFORMAT=$IMAGEFORMAT" >> config/common
+			echo "UBUNTU_IMAGE_ARGS=\"$UBUNTU_IMAGE_ARGS\"" >> config/common
+			# Store model assertion in top dir to get it picked up later as a build artifact
+			env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 model="$MODEL" brand-id=canonical > "$PREFIX".model-assertion
+			echo "Configured ubuntu-image for the following model assertion:"
+			cat "$PREFIX".model-assertion
+			echo "----------------------------------------------------------"
+		else
+			# classic images
+
+			UBUNTU_IMAGE_ARGS=""
+			UBUNTU_IMAGE_ARGS="$UBUNTU_IMAGE_ARGS${PROPOSED:+ --with-proposed}"
+			UBUNTU_IMAGE_ARGS="$UBUNTU_IMAGE_ARGS${EXTRA_PPAS:+ --extra-ppas \"$EXTRA_PPAS\"}"
+
+			git clone git://git.launchpad.net/~canonical-foundations/snap-$MODEL/+git/github-mirror -b classic config/$PREFIX-gadget
+
+			echo "IMAGEFORMAT=$IMAGEFORMAT" >> config/common
+			echo "SUITE=$SUITE" >> config/common
+			echo "UBUNTU_IMAGE_ARGS=\"$UBUNTU_IMAGE_ARGS\"" >> config/common
+
+			echo "Configured ubuntu-image for the following gadget model: $MODEL"
+		fi
 		# Fake finished configuration for lb build
 		mkdir -p .build
 		touch .build/config
@@ -751,7 +770,7 @@
 esac
 
 case $ARCH in
-	armel|armhf)
+	armel|armhf|arm64)
 		KERNEL_FLAVOURS="${SUBARCH:-$KERNEL_FLAVOURS}"
 		case $SUBARCH in
 			dove)
@@ -789,6 +808,13 @@
 				add_package install linux-firmware-raspi2 u-boot-rpi flash-kernel u-boot-tools
 				BINARY_REMOVE_LINUX=false
 				;;
+			raspi3)
+				COMPONENTS='main restricted universe multiverse'
+				KERNEL_FLAVOURS=raspi2
+				add_package install raspi3-firmware u-boot-rpi flash-kernel u-boot-tools
+				BINARY_REMOVE_LINUX=false
+				;;
+
 		esac
 		;;
 esac
@@ -854,7 +880,7 @@
 echo "LB_DISTRIBUTION=\"$SUITE\"" >> config/binary
 
 case $ARCH+$SUBARCH in
-	armhf+raspi2)
+	armhf+raspi2|armhf+raspi3|arm64+raspi3)
 		cat > config/hooks/01-firmware-directory.chroot_early <<EOF
 #!/bin/sh -ex
 mkdir -p /boot/firmware