Merge ~hyask/autopkgtest-cloud:skia/lxd-remote-bos03 into autopkgtest-cloud:master

[Skia]

Based on the experience deploying armhf LXD remotes on bos03, here are the necessary changes that were needed.

[Tim Andersson (andersson123)]

I only have one inline comment, but other than that, this LGTM as long as it doesn't break bos02 or bos01, which I'm presuming it doesn't.

I may also not be the best person to review this, but I think it looks good :)

[Skia (hyask)]

This completely breaks bos02, since the network interface and disk block device name have changed. Still, we decided to drop bos02, so this won't matter much, and even if we did want to go back to bos02, it's just a revert away :-)

[Tim Andersson (andersson123)]

great, well, this is an approval in spirit until the day we decide to fully drop bos02.

review: Approve

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/armhf-lxd.userdata b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/armhf-lxd.userdata
index bb50ada..291715b 100644
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/armhf-lxd.userdata
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/armhf-lxd.userdata
@@ -61,7 +61,6 @@ write_files:
          ipv4.address: 10.0.4.1/24
          ipv4.nat: "true"
          ipv6.address: none
-         raw.dnsmasq: dhcp-option=6,91.189.91.130
        description: ""
        name: lxdbr0
        type: bridge
@@ -79,8 +78,8 @@ write_files:
          security.nesting: "true"
        description: Default LXD profile
        devices:
-         eth0:
-           name: eth0
+         enp1s0:
+           name: enp1s0
            nictype: bridged
            parent: lxdbr0
            type: nic
@@ -100,14 +99,23 @@ write_files:
      systemctl stop snap.lxd.daemon.unix.socket snap.lxd.daemon.service || true
      systemctl start snap.lxd.daemon.unix.socket snap.lxd.daemon.service
      lxd init --preseed < /var/tmp/setup/lxd-init
-     if lxc image info ubuntu-daily:$(distro-info --devel)/armhf >/dev/null 2>&1; then
-         /home/ubuntu/autopkgtest/tools/autopkgtest-build-lxd ubuntu-daily:$(distro-info --devel)/armhf
+     export MIRROR=http://ftpmaster.internal/ubuntu/
+     for r in $({ distro-info --supported; distro-info --supported-esm; } | sort -u | grep -v "$(distro-info --devel)\|trusty"); do
+         command="/home/ubuntu/autopkgtest/tools/autopkgtest-build-lxd ubuntu:$r/armhf"
+         ( sleep 20; $command ) ||  \
+            ( sleep 20; $command ) ||  \
+            ( sleep 20; $command ) || echo "Failed building $r image, please do it manually with 'MIRROR=$MIRROR $command'"
+     done
+     devel="$(distro-info --devel)"
+     if lxc image info ubuntu-daily:$devel/armhf >/dev/null 2>&1; then
+         command="/home/ubuntu/autopkgtest/tools/autopkgtest-build-lxd ubuntu-daily:$devel/armhf"
+         $command || $command || $command || echo "Failed building $devel image, please do it manually with 'MIRROR=$MIRROR $command'"
      else
          # if image for the devel release does not exist then take the latest stable release and upgrade it.
          # see autopkgtest-build-lxd(1) on what the RELEASE environment variable does.
-         MIRROR=http://ftpmaster.internal/ubuntu/ RELEASE=$(distro-info --devel) /home/ubuntu/autopkgtest/tools/autopkgtest-build-lxd ubuntu:$(distro-info --stable)/armhf
+         command="RELEASE=$(distro-info --devel) /home/ubuntu/autopkgtest/tools/autopkgtest-build-lxd ubuntu:$(distro-info --stable)/armhf"
+         $command || $command || $command || echo "Failed building $devel image, please do it manually with 'MIRROR=$MIRROR $command'"
      fi
-     for r in $({ distro-info --supported; distro-info --supported-esm; } | sort -u | grep -v "$(distro-info --devel)\|trusty"); do /home/ubuntu/autopkgtest/tools/autopkgtest-build-lxd ubuntu:$r/armhf; done
      echo "Finished building - rebooting so changed kernel command line comes into effect..."
      reboot
 
@@ -115,18 +123,18 @@ write_files:
 # http://askubuntu.com/questions/446991/gparted-claims-whole-hard-drive-is-unallocated-and-gives-warning-about-gpt-table
 # then create new ~ 100 GB partition and mount it as /srv
 bootcmd:
- - printf 'x\ne\nw\ny\n' | gdisk /dev/sda
- - printf "d\n1\nn\n\n\n+10000M\n\n\nw\ny\n" | gdisk /dev/sda
+ - printf 'x\ne\nw\ny\n' | gdisk /dev/vda
+ - printf "d\n1\nn\n\n\n+10000M\n\n\nw\ny\n" | gdisk /dev/vda
  - partprobe
- - resize2fs /dev/sda1
- - lsblk -dno PARTUUID /dev/sda1 | awk '{ print "GRUB_FORCE_PARTUUID=" $0 }' | tee /etc/default/grub.d/40-force-partuuid.cfg
+ - resize2fs /dev/vda1
+ - lsblk -dno PARTUUID /dev/vda1 | awk '{ print "GRUB_FORCE_PARTUUID=" $0 }' | tee /etc/default/grub.d/40-force-partuuid.cfg
  - update-grub
- - echo 'mkpart primary btrfs 10600MB 100%\n' | parted /dev/sda
- - for timeout in `seq 10`; do [ -e /dev/sda2 ] && break; sleep 1; done
+ - echo 'mkpart primary btrfs 10600MB 100%\n' | parted /dev/vda
+ - for timeout in `seq 10`; do [ -e /dev/vda2 ] && break; sleep 1; done
 
 # use compat_uts_machine=armv7l so that 'linux32 uname -m' shows armv7l, like the buildds
 runcmd:
- - mkfs.btrfs -L autopkgtest /dev/sda2
+ - mkfs.btrfs -L autopkgtest /dev/vda2
  - systemctl stop snap.lxd.daemon.service snap.lxd.daemon.unix.socket
  - echo 'LABEL=autopkgtest   /srv   btrfs defaults 0 0' >> /etc/fstab
  - echo '/srv/var-snap-lxd-common /var/snap/lxd/common none defaults,bind 0 0' >> /etc/fstab
@@ -154,6 +162,6 @@ runcmd:
  - adduser ubuntu lxd
 
  - apt-get install -y --no-install-recommends git
- - su - -c '[ -e autopkgtest ] || git clone https://git.launchpad.net/~ubuntu-release/autopkgtest/+git/development autopkgtest' ubuntu
+ - su - -c '[ -e autopkgtest ] || git clone -b ubuntu/cb2df3e+deb822+prod https://salsa.debian.org/ubuntu-ci-team/autopkgtest autopkgtest' ubuntu
  - su - -c 'cat /var/tmp/setup/ubuntu-crontab | crontab -' ubuntu
  - sh -xe /var/tmp/setup/post-boot-setup.sh &
diff --git a/docs/lxd.rst b/docs/lxd.rst
index 2c86419..54dd744 100644
--- a/docs/lxd.rst
+++ b/docs/lxd.rst
@@ -31,11 +31,13 @@ To deploy a new node, after having sourced the cloud ``.rc`` file (``source ~/.s
     $ NET_ID=$(openstack network show net_prod-proposed-migration --format json | jq -r '."id"')
     $ openstack server create --image $IMAGE --flavor m1.xlarge --nic net-id=$NET_ID --key-name prod-proposed-migration-environment --security-group default --security-group lxd --user-data ~/autopkgtest-cloud/autopkgtest-cloud/tools/armhf-lxd.userdata -- lxd-armhfN
 
-*Note*: on a new environment, you may need to upload the `prod-proposed-migration-environment` SSH key with something like ``openstack keypair create --public-key ~/.ssh/id_ed25519.pub prod-proposed-migration-environment``.
+*Note*: on a new environment, you may need to:
+  * upload the `prod-proposed-migration-environment` SSH key with something like ``openstack keypair create --public-key ~/.ssh/id_ed25519.pub prod-proposed-migration-environment``.
+  * make sure the correct security group is created with the right rules (at least allowing ingress 8443/tcp), with ``mojo/make-lxd-secgroup`` (open that script and adapt it if needed, it's currently not very flexible but easy to tweak to your needs).
 
 In the event that you want to deploy the new node on a specifc host you can use the ``--hint same_host`` argument e.g. ``openstack server create --hint same_host=2e838a71-f6d9-46c4-94f9-dd0c6a2632fe``.
 
-N.B. Wait for it to finish deploying (watch ``/var/log/cloud-init-output.log``) on the newly deployed node. If you do not wait then the unit files for the node on the autopkgtest-lxd-worker will fail as there will be missing images for releases.
+N.B. Wait for it to finish deploying (watch ``/var/log/cloud-init-output.log``) on the newly deployed node. If you do not wait then the unit files for the node on the autopkgtest-lxd-worker will fail as there will be missing images for releases. You can also use ``lxc image list`` to check what was correctly built, and finish the work manually if needed. Commands should be in the cloud-init log.
 
 And you then need to add the IP to the ``autopkgtest-lxd-worker``
 ``lxd-remotes`` section, in the ``service-bundle`` for example: