Merge ~paride/curtin:target_arch into curtin:master

[Paride Legovini]

vmtests: separate arch and target_arch in tests

In tests 'arch' is now the arch of the host system, while target_arch
is the architecture the test targets. By default target_arch = arch.

Modify find_arches() to return all the target arches which not excluded
with arch_skip.

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:a6c1772acca7653955076204ee2a15ce80e549a9
https://jenkins.ubuntu.com/server/job/curtin-ci/3705/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-arm64/3705/
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-ppc64el/3705/
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-s390x/3705/
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=torkoal/3705/

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/curtin-ci/3705//rebuild

review: Approve (continuous-integration)

[Paride Legovini (paride)]

I tested this branch by:

 - running the full test_basic.py on amd64
 - running test_basic.py:XenialGAi386TestBasic on amd64
 - running the full test_basic.py on ppc64el with the
   lp:~legovini/curtin:vmtests-ppc64 adapted and merged in

I'm adding an inline comment too.

[Paride Legovini (paride)]

Also tested:

 - test_old_apt_features.py on amd64 (succeeds)
 - test_old_apt_features.py on ppc64el (fails, but fails
   in the exact same way without this branch applied,
   and I don't think the fix belongs here)

Also tested that the i386 image is not downloaded by vmtest-sync-images on ppc64el. This happened before because of the tests setting 'arch = i386'.

[Ryan Harper (raharper)]

A couple of questions in line.

[Ryan Harper (raharper)]

Thanks as we discussed, the Apt class can't use the base-class target arch since it's being used to build class attributes outside of an __init__ method. And I agree that it's really target_arch value being selected (where target_arch == host_arch). Thanks for clarifying.

review: Approve

diff --git a/tests/vmtests/__init__.py b/tests/vmtests/__init__.py
index 1fe041d..a81003d 100644
--- a/tests/vmtests/__init__.py
+++ b/tests/vmtests/__init__.py
@@ -627,6 +627,7 @@ class VMBaseClass(TestCase):
     # these get set from base_vm_classes
     release = None
     arch = None
+    target_arch = None
     kflavor = None
     krel = None
     distro = None
@@ -653,7 +654,7 @@ class VMBaseClass(TestCase):
         logger.debug('cls.ephemeral_ftype: %s', cls.ephemeral_ftype)
         logger.debug('cls.target_ftype: %s', cls.target_ftype)
         eph_img_verstr, ftypes = get_images(
-            IMAGE_SRC_URL, IMAGE_DIR, cls.distro, cls.release, cls.arch,
+            IMAGE_SRC_URL, IMAGE_DIR, cls.distro, cls.release, cls.target_arch,
             krel=cls.krel if cls.krel else cls.release,
             kflavor=cls.kflavor if cls.kflavor else None,
             subarch=cls.subarch if cls.subarch else None,
@@ -676,7 +677,7 @@ class VMBaseClass(TestCase):
                 IMAGE_SRC_URL, IMAGE_DIR,
                 cls.target_distro,
                 cls.target_release,
-                cls.arch, subarch=cls.subarch if cls.subarch else None,
+                cls.target_arch, subarch=cls.subarch if cls.subarch else None,
                 kflavor=cls.kflavor if cls.kflavor else None,
                 krel=cls.target_krel, sync=CURTIN_VMTEST_IMAGE_SYNC,
                 ftypes=(tftype,))
@@ -912,7 +913,7 @@ class VMBaseClass(TestCase):
             dowait = "--dowait"
 
         # create launch cmd
-        cmd = ["tools/launch", "--arch=" + cls.arch, "-v", dowait,
+        cmd = ["tools/launch", "--arch=" + cls.target_arch, "-v", dowait,
                "--smp=" + cls.get_config_smp(), "--mem=%s" % str(cls.mem)]
         if not cls.interactive:
             cmd.extend(["--silent", "--power=off"])
@@ -1317,7 +1318,7 @@ class VMBaseClass(TestCase):
                ["--", "-smp",  cls.get_config_smp(), "-m", str(cls.mem)])
 
         if not cls.interactive:
-            if cls.arch == 's390x':
+            if cls.target_arch == 's390x':
                 cmd.extend([
                     "-nographic", "-nodefaults", "-chardev",
                     "file,path=%s,id=charconsole0" % cls.boot_log,
@@ -1722,7 +1723,7 @@ class VMBaseClass(TestCase):
             raise SkipTest("kernel-img.conf not needed in non-ubuntu releases")
         kconf = 'kernel-img.conf'
         self.output_files_exist([kconf])
-        if self.arch in ['i386', 'amd64']:
+        if self.target_arch in ['i386', 'amd64']:
             self.check_file_strippedline(kconf, 'link_in_boot = no')
         else:
             self.check_file_strippedline(kconf, 'link_in_boot = yes')
@@ -1845,7 +1846,7 @@ class PsuedoVMBaseClass(VMBaseClass):
            kernel, initrd, tarball."""
 
         def get_psuedo_path(name):
-            return os.path.join(IMAGE_DIR, cls.release, cls.arch, name)
+            return os.path.join(IMAGE_DIR, cls.release, cls.target_arch, name)
 
         return {
             'squashfs': get_psuedo_path('psuedo-squashfs'),
diff --git a/tests/vmtests/helpers.py b/tests/vmtests/helpers.py
index 6dddcc6..835baed 100644
--- a/tests/vmtests/helpers.py
+++ b/tests/vmtests/helpers.py
@@ -8,6 +8,8 @@ import signal
 import threading
 from unittest import TestLoader
 
+from curtin.util import get_platform_arch
+
 
 class Command(object):
     """
@@ -135,14 +137,17 @@ def find_testcases(**kwargs):
 
 def find_arches():
     """
-    Return a list of uniq arch values from test cases
+    Return a list of uniq target arch values from test cases
     """
-    arches = []
+    arch = get_platform_arch()
+    target_arches = []
     for test_case in find_testcases():
-        arch = getattr(test_case, 'arch', None)
-        if arch and arch not in arches:
-            arches.append(arch)
-    return arches
+        if arch in getattr(test_case, 'arch_skip', None):
+            continue
+        target_arch = getattr(test_case, 'target_arch', None)
+        if target_arch and target_arch not in target_arches:
+            target_arches.append(target_arch)
+    return target_arches
 
 
 def find_releases_by_distro():
diff --git a/tests/vmtests/releases.py b/tests/vmtests/releases.py
index 6cc0d8b..16302e8 100644
--- a/tests/vmtests/releases.py
+++ b/tests/vmtests/releases.py
@@ -6,6 +6,7 @@ from curtin.util import get_platform_arch
 class _ReleaseBase(object):
     repo = "maas-daily"
     arch = get_platform_arch()
+    target_arch = arch
     mem = "1024"
 
 
diff --git a/tests/vmtests/test_basic.py b/tests/vmtests/test_basic.py
index 4d5008c..c15dd50 100644
--- a/tests/vmtests/test_basic.py
+++ b/tests/vmtests/test_basic.py
@@ -264,7 +264,7 @@ class Centos66BionicTestBasic(centos_relbase.centos66_bionic,
 
 class XenialGAi386TestBasic(relbase.xenial_ga, TestBasicAbs):
     __test__ = True
-    arch = 'i386'
+    target_arch = 'i386'
 
 
 class XenialGATestBasic(relbase.xenial_ga, TestBasicAbs):
diff --git a/tests/vmtests/test_old_apt_features.py b/tests/vmtests/test_old_apt_features.py
index a114bf4..5a5415c 100644
--- a/tests/vmtests/test_old_apt_features.py
+++ b/tests/vmtests/test_old_apt_features.py
@@ -56,11 +56,12 @@ class TestOldAptAbs(VMBaseClass):
         exit 0
         """)]
     arch = util.get_architecture()
-    if arch in ['amd64', 'i386']:
+    target_arch = arch
+    if target_arch in ['amd64', 'i386']:
         conf_file = "examples/tests/test_old_apt_features.yaml"
         exp_mirror = "http://us.archive.ubuntu.com/ubuntu"
         exp_secmirror = "http://archive.ubuntu.com/ubuntu"
-    if arch in ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el']:
+    if target_arch in ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el']:
         conf_file = "examples/tests/test_old_apt_features_ports.yaml"
         exp_mirror = "http://ports.ubuntu.com/ubuntu-ports"
         exp_secmirror = "http://ports.ubuntu.com/ubuntu-ports"
diff --git a/tests/vmtests/test_zfsroot.py b/tests/vmtests/test_zfsroot.py
index c0cfe01..59bb64b 100644
--- a/tests/vmtests/test_zfsroot.py
+++ b/tests/vmtests/test_zfsroot.py
@@ -74,7 +74,7 @@ class UnsupportedZfs(VMBaseClass):
 class XenialGAi386TestZfsRoot(relbase.xenial_ga, TestZfsRootAbs,
                               UnsupportedZfs):
     __test__ = True
-    arch = 'i386'
+    target_arch = 'i386'
 
 
 class XenialGATestZfsRoot(relbase.xenial_ga, TestZfsRootAbs):
@@ -115,7 +115,7 @@ class XenialGATestZfsRootFsType(relbase.xenial_ga, TestZfsRootFsTypeAbs):
 class XenialGAi386TestZfsRootFsType(relbase.xenial_ga, TestZfsRootFsTypeAbs,
                                     UnsupportedZfs):
     __test__ = True
-    arch = 'i386'
+    target_arch = 'i386'
 
 
 class BionicTestZfsRootFsType(relbase.bionic, TestZfsRootFsTypeAbs):