Merge ~raharper/curtin:feature/vmtest-add-centos-from-bionic into curtin:master

[Ryan Harper]

vmtest: add Centos66/Centos70 FromBionic release and re-add tests

Newer MAAS will use Bionic to deploy Centos and curtin until now
did not test this path. Critically, Centos66 has issues with ext4
features that are enabled by default in Bionic and newer, namely
'64bit' and 'metadata_csum'. These features break centos66 grub
probing which results in failed deployment, LP: #1775424.

Add additional Centos66/70 tests to verify installing from Bionic
is successful. For Centos66, we test the known workaround which is
to use 'ext3' as the root filesystem, and we introduce a test-case
which recreates the failure (test_simple.py:Centos66BionicTestSimple)
but leave it disabled by default.

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:8dc196ea54bfb3758b3c81f4faa7e6edecaa5411
https://jenkins.ubuntu.com/server/job/curtin-ci/1289/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-arm64/1289
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-ppc64el/1289
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-s390x/1289
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=torkoal/1289

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/curtin-ci/1289/rebuild

review: Approve (continuous-integration)

[Dan Watkins (oddbloke)]

One very minor inline comment, and you've also got "additiona" in your commit message. I'd be happy to merge as-is, but I'll leave it to you to Approve in case you do want to address those.

review: Approve

[Ryan Harper (raharper)]

Thanks.

2720f6b... by Ryan Harper on 2019-06-05

clarify btrfs-debug-tree comment

[Dan Watkins (oddbloke)]

Oh right, forgot I could edit the commit message on Launchpad MPs. >.< Thanks!

diff --git a/examples/tests/centos6_basic.yaml b/examples/tests/centos6_basic.yaml
new file mode 100644
index 0000000..90fc584
--- /dev/null
+++ b/examples/tests/centos6_basic.yaml
@@ -0,0 +1,101 @@
+showtrace: true
+storage:
+    version: 1
+    config:
+      - id: sda
+        type: disk
+        ptable: msdos
+        model: QEMU HARDDISK
+        serial: disk-a
+        name: main_disk_with_in/\&valid@#dname
+        wipe: superblock
+        grub_device: true
+      - id: sda1
+        type: partition
+        number: 1
+        size: 3GB
+        device: sda
+        flag: boot
+      - id: sda2
+        type: partition
+        number: 2
+        size: 1GB
+        device: sda
+      - id: sda3
+        type: partition
+        number: 3
+        size: 1GB
+        device: sda
+        name: swap
+      - id: sda1_root
+        type: format
+        fstype: ext3
+        volume: sda1
+        label: 'cloudimg-rootfs'
+      - id: sda2_home
+        type: format
+        fstype: ext4
+        volume: sda2
+      - id: sda3_swap
+        type: format
+        fstype: swap
+        volume: sda3
+      - id: sda1_mount
+        type: mount
+        path: /
+        device: sda1_root
+      - id: sda2_mount
+        type: mount
+        path: /home
+        device: sda2_home
+      - id: sparedisk_id
+        type: disk
+        serial: disk-b
+        name: sparedisk
+        wipe: superblock
+      - id: sparedisk_fat_fmt_id
+        type: format
+        fstype: fat32
+        volume: sparedisk_id
+      - id: btrfs_disk_id
+        type: disk
+        serial: disk-c
+        name: btrfs_volume
+        wipe: superblock
+      - id: btrfs_disk_fmt_id
+        type: format
+        fstype: btrfs
+        volume: btrfs_disk_id
+      - id: btrfs_disk_mnt_id
+        type: mount
+        path: /btrfs
+        options: 'defaults,noatime'
+        device: btrfs_disk_fmt_id
+      - id: pnum_disk
+        type: disk
+        serial: disk-d
+        name: pnum_disk
+        wipe: superblock
+        ptable: gpt
+      - id: pnum_disk_p1
+        type: partition
+        number: 1
+        size: 1GB
+        device: pnum_disk
+      - id: pnum_disk_p2
+        type: partition
+        number: 2
+        size: 8MB
+        device: pnum_disk
+        flag: prep
+        wipe: zero
+        name: prep
+      - id: pnum_disk_p3
+        type: partition
+        number: 10
+        size: 1GB
+        device: pnum_disk
+      - id: swap_mnt
+        type: mount
+        path: "none"
+        device: sda3_swap
diff --git a/tests/vmtests/releases.py b/tests/vmtests/releases.py
index 1a92412..68deae1 100644
--- a/tests/vmtests/releases.py
+++ b/tests/vmtests/releases.py
@@ -36,6 +36,13 @@ class _Centos70FromXenialBase(_CentosFromUbuntuBase):
     target_release = "centos70"
 
 
+class _Centos70FromBionicBase(_CentosFromUbuntuBase):
+    # release for boot
+    release = "bionic"
+    # release for target
+    target_release = "centos70"
+
+
 class _UbuntuCore16FromXenialBase(_UbuntuCoreUbuntuBase):
     # release for boot
     release = "xenial"
@@ -49,6 +56,11 @@ class _Centos66FromXenialBase(_CentosFromUbuntuBase):
     target_release = "centos66"
 
 
+class _Centos66FromBionicBase(_CentosFromUbuntuBase):
+    release = "bionic"
+    target_release = "centos66"
+
+
 class _PreciseBase(_UbuntuBase):
     release = "xenial"
     target_release = "precise"
@@ -149,6 +161,8 @@ class _Releases(object):
 class _CentosReleases(object):
     centos70_xenial = _Centos70FromXenialBase
     centos66_xenial = _Centos66FromXenialBase
+    centos70_bionic = _Centos70FromBionicBase
+    centos66_bionic = _Centos66FromBionicBase
 
 
 class _UbuntuCoreReleases(object):
diff --git a/tests/vmtests/test_basic.py b/tests/vmtests/test_basic.py
index 92971b3..d65a8c1 100644
--- a/tests/vmtests/test_basic.py
+++ b/tests/vmtests/test_basic.py
@@ -37,6 +37,10 @@ class TestBasicAbs(VMBaseClass):
         f="btrfs_uuid_diskc"
         if command -v btrfs-debug-tree >/dev/null; then
            btrfs-debug-tree -r $dev | awk '/^uuid/ {print $2}' | grep "-"
+           # btrfs-debug-tree fails in centos66, use btrfs-show instead
+           if [ "$?" != "0" ]; then
+               btrfs-show $dev | awk '/uuid/ {print $4}'
+           fi
         else
            btrfs inspect-internal dump-super $dev |
                awk '/^dev_item.fsid/ {print $2}'
@@ -46,7 +50,8 @@ class TestBasicAbs(VMBaseClass):
         diskd=$(readlink -f /dev/disk/by-id/*-disk-d)
         cmp --bytes=8388608 /dev/zero ${diskd}2; echo "$?" > cmp_prep.out
         # extract partition info
-        udevadm info --export --query=property ${diskd}2 | cat >udev_info.out
+        udevadm info --export --query=property --name=${diskd}2 |
+            cat >udev_info.out
 
         exit 0
         """)]
@@ -81,6 +86,9 @@ class TestBasicAbs(VMBaseClass):
         return kname
 
     def _test_ptable(self, blkid_output, expected):
+        if self.target_release == "centos66":
+            raise SkipTest("No PTTYPE blkid output on Centos66")
+
         if not blkid_output:
             raise RuntimeError('_test_ptable requires blkid output file')
 
@@ -138,6 +146,8 @@ class TestBasicAbs(VMBaseClass):
         self.assertEqual(kname_uuid, btrfs_uuid)
 
     def _test_partition_is_prep(self, info_file):
+        if self.target_release == "centos66":
+            raise SkipTest("Cannot detect PReP partitions in Centos66")
         udev_info = self.load_collect_file(info_file).rstrip()
         entry_type = ''
         for line in udev_info.splitlines():
@@ -234,6 +244,24 @@ class Centos70XenialTestBasic(centos_relbase.centos70_xenial,
     __test__ = True
 
 
+class Centos70BionicTestBasic(centos_relbase.centos70_bionic,
+                              CentosTestBasicAbs):
+    __test__ = True
+
+
+class Centos66XenialTestBasic(centos_relbase.centos66_xenial,
+                              CentosTestBasicAbs):
+    __test__ = True
+
+
+class Centos66BionicTestBasic(centos_relbase.centos66_bionic,
+                              CentosTestBasicAbs):
+    # Centos66 cannot handle ext4 defaults in Bionic (64bit,meta_csum)
+    # this conf defaults to ext3
+    conf_file = "examples/tests/centos6_basic.yaml"
+    __test__ = True
+
+
 class XenialGAi386TestBasic(relbase.xenial_ga, TestBasicAbs):
     __test__ = True
     arch = 'i386'
diff --git a/tests/vmtests/test_simple.py b/tests/vmtests/test_simple.py
index bfac4e9..9c31b65 100644
--- a/tests/vmtests/test_simple.py
+++ b/tests/vmtests/test_simple.py
@@ -1,6 +1,6 @@
 # This file is part of curtin. See LICENSE file for copyright and license info.
 
-from . import VMBaseClass
+from . import VMBaseClass, SkipTest
 from .releases import base_vm_classes as relbase
 from .releases import centos_base_vm_classes as centos_relbase
 
@@ -21,10 +21,24 @@ class TestSimple(VMBaseClass):
         """)]
 
 
-class Centos70TestSimple(centos_relbase.centos70_xenial, TestSimple):
+class Centos70XenialTestSimple(centos_relbase.centos70_xenial, TestSimple):
     __test__ = True
 
 
+class Centos70BionicTestSimple(centos_relbase.centos70_bionic, TestSimple):
+    __test__ = True
+
+
+class Centos66XenialTestSimple(centos_relbase.centos66_xenial, TestSimple):
+    __test__ = True
+
+
+class Centos66BionicTestSimple(centos_relbase.centos66_bionic, TestSimple):
+    __test__ = False
+    # LP: #1775424
+    raise SkipTest('Centos66 fails with Bionic Ephemeral ext4 features')
+
+
 class XenialTestSimple(relbase.xenial, TestSimple):
     __test__ = True