Merge ~ubuntu-release/autopkgtest/+git/development:cross-arch-testing into ~ubuntu-release/autopkgtest/+git/development:master

[Steve Langasek]

A first pass at implementing a -a $arch argument to autopkgtest to support cross-arch testing

c4fc700... by Steve Langasek on 2019-11-29

coding style

[Julian Andres Klode (juliank)]

The commits look good to me from reading them, except that one comment below. Not performed any real-world testing yet.

[Julian Andres Klode (juliank)]

The code seems to do the right thing when testing it, but see the previous note about the missing = in the man page.

review: Approve

[Brian Murray (brian-murray)]

All the commit ids in this merge proposal exist in the master branch of autopkgtest so I'm setting this to merged.

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f28fa86..b9e1492 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,9 +17,9 @@ tests-sid:
     - apt-get install -y apt-utils autodep8 build-essential debhelper libdpkg-perl procps python3 python3-debian
     - tests/autopkgtest NullRunner NullRunnerRoot ChrootRunner
 
-tests-stretch:
+tests-stable:
   stage: test
-  image: debian:stretch
+  image: debian:stable
   script:
     - apt-get update
     - apt-get install -y apt-utils autodep8 build-essential debhelper libdpkg-perl procps python3 python3-debian
diff --git a/debian/control b/debian/control
index 6b347f0..c37b4ee 100644
--- a/debian/control
+++ b/debian/control
@@ -1,39 +1,41 @@
 Source: autopkgtest
 Maintainer: Debian CI team <[e-mail]>
-Uploaders: Ian Jackson <[e-mail]>, Martin Pitt <[e-mail]>, Antonio Terceiro <[e-mail]>, Paul Gevers <[e-mail]>
+Uploaders: Ian Jackson <[e-mail]>,
+           Martin Pitt <[e-mail]>,
+           Antonio Terceiro <[e-mail]>,
+           Paul Gevers <[e-mail]>
 Section: devel
 Priority: optional
 Standards-Version: 4.4.0
 Build-Depends: debhelper (>= 9),
- python3 (>= 3.1),
- python3-mock,
- python3-debian,
- python3-docutils,
- pyflakes3 | pyflakes,
- procps,
- pycodestyle | pep8,
+               procps,
+               pycodestyle | pep8,
+               pyflakes3 | pyflakes,
+               python3 (>= 3.1),
+               python3-debian,
+               python3-docutils,
+               python3-mock
 Vcs-Git: https://salsa.debian.org/ci-team/autopkgtest.git
 Vcs-Browser: https://salsa.debian.org/ci-team/autopkgtest
 
 Package: autopkgtest
 Architecture: all
-Depends: python3,
- python3-debian,
- apt-utils,
- libdpkg-perl,
- procps,
- ${misc:Depends}
+Depends: apt-utils,
+         libdpkg-perl,
+         procps,
+         python3,
+         python3-debian,
+         ${misc:Depends}
 Recommends: autodep8
-Suggests:
- lxc,
- lxd,
- ovmf,
- qemu-efi-aarch64,
- qemu-efi-arm,
- qemu-system,
- qemu-utils,
- schroot,
- vmdb2,
+Suggests: lxc,
+          lxd,
+          ovmf,
+          qemu-efi-aarch64,
+          qemu-efi-arm,
+          qemu-system,
+          qemu-utils,
+          schroot,
+          vmdb2
 Breaks: debci (<< 1.7~)
 Description: automatic as-installed testing for Debian packages
  autopkgtest runs tests on binary packages.  The tests are run on the
diff --git a/debian/tests/control b/debian/tests/control
index a59b3ef..26f4a18 100644
--- a/debian/tests/control
+++ b/debian/tests/control
@@ -1,8 +1,5 @@
 Tests: autopkgtest
-Depends: autopkgtest,
- autodep8,
- build-essential,
- debhelper (>= 7)
+Depends: autodep8, autopkgtest, build-essential, debhelper (>= 7)
 Restrictions: needs-root
 Tests-Directory: tests
 
@@ -15,9 +12,9 @@ Depends: autopkgtest
 
 Tests: lxd
 Depends: autopkgtest,
- lxd,
- build-essential,
- debhelper (>= 7),
- fakeroot,
- iptables
+         build-essential,
+         debhelper (>= 7),
+         fakeroot,
+         iptables,
+         lxd
 Restrictions: isolation-machine, needs-root, allow-stderr
diff --git a/lib/adt_testbed.py b/lib/adt_testbed.py
index 4fe8874..ff1a0ff 100644
--- a/lib/adt_testbed.py
+++ b/lib/adt_testbed.py
@@ -395,7 +395,8 @@ class Testbed:
             self._opened(pl)
         self.modified = False
 
-    def install_deps(self, deps_new, recommends, shell_on_failure=False, synth_deps=[]):
+    def install_deps(self, deps_new, recommends, shell_on_failure=False,
+                     synth_deps=[], cross_arch=None):
         '''Install dependencies into testbed'''
         adtlog.debug('install_deps: deps_new=%s, recommends=%s' % (deps_new, recommends))
 
@@ -403,7 +404,24 @@ class Testbed:
         self.recommends_installed = recommends
         if not deps_new:
             return
-        self.satisfy_dependencies_string(', '.join(deps_new), 'install-deps', recommends, shell_on_failure=shell_on_failure, synth_deps=synth_deps)
+        if cross_arch:
+            # mock up a .dsc with our dependencies string as our build-deps
+            dsc = TempPath(self, 'autopkgtest-satdep.dsc')
+            with open(dsc.host, 'w', encoding='UTF-8') as f_dsc:
+                f_dsc.write("Source: autopkgtest-satdep\n"
+                            "Binary: autopkgtest-satdep\n"
+                            "Architecture: any all\n"
+                            "Build-Depends: " + ', '.join(deps_new) + "\n")
+            dsc.copydown()
+
+            # feed the result to apt-get build-dep
+            self.satisfy_build_deps(dsc.tb, cross_arch,
+                                    recommends, shell_on_failure)
+            # Now handle the synthetic deps as well
+            self.install_apt('', recommends, shell_on_failure, synth_deps)
+
+        else:
+            self.satisfy_dependencies_string(', '.join(deps_new), 'install-deps', recommends, shell_on_failure=shell_on_failure, synth_deps=synth_deps)
 
     def needs_reset(self):
         # show what caused a reset
@@ -1021,6 +1039,105 @@ fi
         if self.execute(['sh', adtlog.verbosity >= 2 and '-exc' or '-ec', script], kind='install')[0] != 0:
             self.bomb('Failed to update click AppArmor rules')
 
+    def _run_apt_build_dep(self, what, prefix, recommends, ignorerc=False):
+        '''actually run apt-get build-dep'''
+
+        # capture status-fd to stderr
+        (rc, _, serr) = self.execute(['/bin/sh', '-ec', '%s apt-get build-dep '
+                                      '--assume-yes %s '
+                                      '-o APT::Status-Fd=3 '
+                                      '-o APT::Install-Recommends=%s '
+                                      '-o Dpkg::Options::=--force-confnew '
+                                      '-o Debug::pkgProblemResolver=true 3>&2 2>&1' %
+                                      (prefix, what, recommends)],
+                                     kind='install', stderr=subprocess.PIPE)
+        if not ignorerc and rc != 0:
+            adtlog.debug('apt-get build-dep %s failed; status-fd:\n%s' % (what, serr))
+            # check if apt failed during package download, which might be a
+            # transient error, so retry
+            if 'dlstatus:' in serr and 'pmstatus:' not in serr:
+                raise adtlog.AptDownloadError
+            else:
+                raise adtlog.AptPermanentError
+
+    def satisfy_build_deps(self, source_pkg, architecture=None,
+                           recommends=False, shell_on_failure=False):
+        '''Install build-dependencies into testbed with apt-get build-dep
+
+        This requires root privileges and a writable file system.
+        '''
+        # check if we can use apt-get
+        can_apt_get = False
+        if 'root-on-testbed' in self.caps:
+            rc = self.execute(['test', '-w', '/var/lib/dpkg/status'])[0]
+            if rc == 0:
+                can_apt_get = True
+        adtlog.debug('can use apt-get on testbed: %s' % can_apt_get)
+
+        if not can_apt_get:
+            self.bomb('no root on testbed, unsupported when specifying target architecture')
+
+        what = source_pkg
+        if architecture:
+            what += ' -a ' + architecture
+        # install the build-dependencies for the specified source (which can
+        # be a source package name, or a path); our apt pinning is not
+        # very clever wrt. resolving transitional dependencies in the pocket,
+        # so we might need to retry without pinning
+        download_fail_retries = 3
+        while True:
+            rc = 0
+            try:
+                self._run_apt_build_dep(what, ' '.join(self.eatmydata_prefix), recommends)
+
+            # check if apt failed during package download, which might be a
+            # transient error, so retry
+            except adtlog.AptDownloadError:
+                download_fail_retries -= 1
+                if download_fail_retries > 0:
+                    adtlog.warning('apt failed to download packages, retrying in 10s...')
+                    time.sleep(10)
+                    continue
+                else:
+                    self.bomb('apt repeatedly failed to download packages')
+
+            except adtlog.AptPermanentError:
+                rc = -1
+                if shell_on_failure:
+                    self.run_shell()
+
+            if rc != 0:
+                if self.apt_pin_for_releases and self.enable_apt_fallback:
+                    release = self.apt_pin_for_releases.pop()
+                    adtlog.warning('Test dependencies are unsatisfiable using apt pinning. '
+                                   'Retrying using all packages from %s' % release)
+                    self.check_exec(['/bin/sh', '-ec', 'rm /etc/apt/preferences.d/autopkgtest-' + release])
+                    if not self.apt_pin_for_releases:
+                        self.check_exec(['/bin/sh', '-ec', 'rm -f /etc/apt/preferences.d/autopkgtest-default-release'])
+                    continue
+
+                adtlog.warning('Test dependencies are unsatisfiable - calling '
+                               'apt install on test deps directly for further '
+                               'data about failing dependencies in test logs')
+                self._run_apt_build_dep('--simulate ' + what,
+                                        ' '.join(self.eatmydata_prefix),
+                                        recommends, ignorerc=True)
+
+                if shell_on_failure:
+                    self.run_shell()
+                if self.enable_apt_fallback:
+                    self.badpkg('Test dependencies are unsatisfiable. A common reason is '
+                                'that your testbed is out of date with respect to the '
+                                'archive, and you need to use a current testbed or run '
+                                'apt-get update or use -U.')
+                else:
+                    self.badpkg('Test dependencies are unsatisfiable. A common reason is '
+                                'that the requested apt pinning prevented dependencies '
+                                'from the non-default suite to be installed. In that '
+                                'case you need to add those dependencies to the pinning '
+                                'list.')
+            break
+
     def satisfy_dependencies_string(self, deps, what, recommends=False,
                                     build_dep=False, shell_on_failure=False, synth_deps=[]):
         '''Install dependencies from a string into the testbed'''
diff --git a/lib/autopkgtest_args.py b/lib/autopkgtest_args.py
index 7fea2d8..9edb4a8 100644
--- a/lib/autopkgtest_args.py
+++ b/lib/autopkgtest_args.py
@@ -210,6 +210,11 @@ for details.'''
                         help='Run tests from already installed click package '
                         '(e. g. "com.example.myapp"), from specified click '
                         'source directory or manifest\'s x-source.')
+    g_test.add_argument('-a', '--architecture', metavar='ARCH',
+                        help='run tests for (and when asked, build binaries '
+                        'for) ARCH instead of the testbed host architecture. '
+                        'Assumes ARCH is available as a foreign architecture '
+                        'on the testbed.')
     g_test.add_argument('packages', nargs='*',
                         help='testsrc source package and testbinary packages as above')
 
diff --git a/lib/testdesc.py b/lib/testdesc.py
index 84d24db..eb2ffd4 100644
--- a/lib/testdesc.py
+++ b/lib/testdesc.py
@@ -239,7 +239,7 @@ def _debian_check_unknown_fields(name, record):
         raise Unsupported(name, 'unknown field %s' % unknown_keys.pop())
 
 
-def _debian_packages_from_source(srcdir):
+def _debian_packages_from_source(srcdir, cross_arch=None):
     packages = []
     packages_no_arch = []
 
@@ -252,10 +252,17 @@ def _debian_packages_from_source(srcdir):
                 st.get('Package-type', 'deb') != 'deb':
             continue
         arch = st['Architecture']
-        if arch in ('all', 'any'):
+        qual_pkg = st['Package']
+        # take care to emit an arch qualifier only for arch-dependent
+        # packages, not for arch: all ones
+        if cross_arch:
+            qual_pkg += ':' + cross_arch
+        if arch == 'all':
             packages.append(st['Package'])
+        elif arch == 'any':
+            packages.append(qual_pkg)
         else:
-            packages.append('%s [%s]' % (st['Package'], arch))
+            packages.append('%s [%s]' % (qual_pkg, arch))
         packages_no_arch.append(st['Package'])
 
     return (packages, packages_no_arch)
@@ -353,7 +360,8 @@ def _synthesize_deps(dep, testbed_arch):
         return None
 
 
-def _parse_debian_depends(testname, dep_str, srcdir, testbed_arch):
+def _parse_debian_depends(testname, dep_str, srcdir, testbed_arch,
+                          cross_arch=None):
     '''Parse Depends: line in a Debian package
 
     Split dependencies (comma separated), validate their syntax, and expand @
@@ -364,7 +372,12 @@ def _parse_debian_depends(testname, dep_str, srcdir, testbed_arch):
     '''
     deps = []
     synthdeps = []
-    (my_packages, my_packages_no_arch) = _debian_packages_from_source(srcdir)
+    (my_packages, my_packages_no_arch) = _debian_packages_from_source(srcdir,
+                                                                      cross_arch=cross_arch)
+    if cross_arch:
+        target_arch = cross_arch
+    else:
+        target_arch = testbed_arch
     for alt_group_str in dep_str.split(','):
         alt_group_str = alt_group_str.strip()
         if not alt_group_str:
@@ -375,7 +388,7 @@ def _parse_debian_depends(testname, dep_str, srcdir, testbed_arch):
             for d in my_packages:
                 adtlog.debug('synthesised dependency %s' % d)
                 deps.append(d)
-                s = _synthesize_deps(d, testbed_arch)
+                s = _synthesize_deps(d, target_arch)
                 if s:
                     synthdeps.append(s)
         elif alt_group_str == '@builddeps@':
@@ -389,8 +402,10 @@ def _parse_debian_depends(testname, dep_str, srcdir, testbed_arch):
                 if pkg not in my_packages_no_arch:
                     synthdep_alternatives = []
                     break
-                s = _synthesize_deps(dep, testbed_arch)
+                s = _synthesize_deps(dep, target_arch)
                 if s:
+                    if cross_arch:
+                        s += ':' + cross_arch
                     synthdep_alternatives.append(s)
             if synthdep_alternatives:
                 adtlog.debug('marked alternatives %s as a synthesised dependency' % synthdep_alternatives)
@@ -398,6 +413,9 @@ def _parse_debian_depends(testname, dep_str, srcdir, testbed_arch):
                     synthdeps.append(synthdep_alternatives)
                 else:
                     synthdeps.append(synthdep_alternatives[0])
+            if cross_arch and alt_group_str + ':' + cross_arch in my_packages:
+                adtlog.debug('%s is from our source package, adding arch qualifier for cross-testing' % s)
+                alt_group_str += ':' + cross_arch
             deps.append(alt_group_str)
 
     return (deps, synthdeps)
@@ -429,7 +447,7 @@ def _autodep8(srcdir):
 
 
 def parse_debian_source(srcdir, testbed_caps, testbed_arch, control_path=None,
-                        auto_control=True):
+                        auto_control=True, cross_arch=None):
     '''Parse test descriptions from a Debian DEP-8 source dir
 
     You can specify an alternative path for the control file (default:
@@ -503,7 +521,7 @@ def parse_debian_source(srcdir, testbed_caps, testbed_arch, control_path=None,
                     test_names[0],
                     record.get('Depends', '@'),
                     srcdir,
-                    testbed_arch)
+                    testbed_arch, cross_arch=cross_arch)
                 if 'Test-command' in record:
                     raise InvalidControl('*', 'Only one of "Tests" or '
                                          '"Test-Command" may be given')
@@ -529,7 +547,7 @@ def parse_debian_source(srcdir, testbed_caps, testbed_arch, control_path=None,
                     command,
                     record.get('Depends', '@'),
                     srcdir,
-                    testbed_arch)
+                    testbed_arch, cross_arch=cross_arch)
                 if feature_test_name is None:
                     command_counter += 1
                     name = 'command%i' % command_counter
diff --git a/runner/autopkgtest b/runner/autopkgtest
index 93c6e61..f1e2e8f 100755
--- a/runner/autopkgtest
+++ b/runner/autopkgtest
@@ -159,7 +159,10 @@ def run_tests(tests, tree):
         binaries.publish()
         doTest = True
         try:
-            testbed.install_deps(t.depends, 'needs-recommends' in t.restrictions, opts.shell_fail, t.synth_depends)
+            testbed.install_deps(t.depends,
+                                 'needs-recommends' in t.restrictions,
+                                 opts.shell_fail, t.synth_depends,
+                                 opts.architecture)
         except adtlog.BadPackageError as e:
             if 'skip-not-installable' in t.restrictions:
                 errorcode |= 2
@@ -388,6 +391,12 @@ def build_source(kind, arg, built_binaries):
         return tests_tree
 
     elif kind == 'apt-source':
+        # Make sure we are selecting the binaries based on the actual target
+        # architecture, not necessarily the testbed architecture
+        if opts.architecture:
+            arch = opts.architecture
+        else:
+            arch = testbed.dpkg_arch
         # The default is to determine the version for "apt-get source
         # pkg=version" that conforms to the current apt pinning. We only
         # consider binaries which are shipped in all available versions,
@@ -443,7 +452,7 @@ pkgs=$(echo "$pkgs\n" | awk "
     if (foundarch == 0 || archmatch == 1) thissrc[\\$1] = 1;
     next }
   { if (!inlist) next;
-    inlist=0;''' % {'src': arg, 'arch': testbed.dpkg_arch}
+    inlist=0;''' % {'src': arg, 'arch': arch}
 
         create_command_part2_check_all_pkgs = '''
     remaining=0;
@@ -580,24 +589,8 @@ dpkg-source -x %(src)s_*.dsc src >/dev/null''' % {'src': arg}
         if kind not in ['dsc', 'apt-source']:
             testbed.install_deps([], False)
 
-        if kind in ('apt-source', 'git-source'):
-            # we need to get the downloaded debian/control from the testbed, so
-            # that we can avoid calling "apt-get build-dep" and thus
-            # introducing a second mechanism for installing build deps
-            pkg_control = adt_testbed.Path(testbed,
-                                           os.path.join(tmp, 'apt-control'),
-                                           os.path.join(result_pwd, 'debian/control'), False)
-            pkg_control.copyup()
-            dsc = pkg_control.host
-
-        with open(dsc, encoding='UTF-8') as f:
-            d = deb822.Deb822(sequence=f)
-            bd = d.get('Build-Depends', '')
-            bdi = d.get('Build-Depends-Indep', '')
-            bda = d.get('Build-Depends-Arch', '')
-
         # determine build command and build-essential packages
-        build_essential = ['build-essential']
+        build_essential = ['build-essential:native']
         assert testbed.nproc
         dpkg_buildpackage = 'DEB_BUILD_OPTIONS="parallel=%s $DEB_BUILD_OPTIONS" dpkg-buildpackage -us -uc -b' % (
             opts.build_parallel or testbed.nproc)
@@ -607,8 +600,47 @@ dpkg-source -x %(src)s_*.dsc src >/dev/null''' % {'src': arg}
             if testbed.user or 'root-on-testbed' not in testbed.caps:
                 build_essential += ['fakeroot']
 
-        testbed.satisfy_dependencies_string(bd + ', ' + bdi + ', ' + bda + ', ' + ', '.join(build_essential), arg,
-                                            build_dep=True, shell_on_failure=opts.shell_fail)
+        if opts.architecture:
+            dpkg_buildpackage += ' -a' + opts.architecture
+            build_essential += ['crossbuild-essential-%s:native' % opts.architecture]
+            # apt-get build-dep is the best option here, but we don't call
+            # it unconditionally because it doesn't take a path to a source
+            # package as an option in very old releases; so for compatibility
+            # only use it when we need its multiarch build-dep resolution
+            # support.
+            # This is supported in apt 1.1~exp2 and newer, which covers
+            # Debian oldstable (at time of writing) and Ubuntu 16.04 and
+            # newer, so once we can rely on this version of apt everywhere,
+            # the old build-dep resolver code should be deprecated.
+            all_build_deps = ''
+            testbed.satisfy_build_deps(result_pwd,
+                                       architecture=opts.architecture,
+                                       shell_on_failure=opts.shell_fail)
+        else:
+            if kind in ('apt-source', 'git-source'):
+                # we need to get the downloaded debian/control from the
+                # testbed, so that we can avoid calling "apt-get build-dep"
+                # and thus introducing a second mechanism for installing
+                # build deps
+                pkg_control = adt_testbed.Path(testbed,
+                                               os.path.join(tmp, 'apt-control'),
+                                               os.path.join(result_pwd, 'debian/control'), False)
+                pkg_control.copyup()
+                dsc = pkg_control.host
+
+            with open(dsc, encoding='UTF-8') as f:
+                d = deb822.Deb822(sequence=f)
+                bd = d.get('Build-Depends', '')
+                bdi = d.get('Build-Depends-Indep', '')
+                bda = d.get('Build-Depends-Arch', '')
+
+            all_build_deps = bd + ', ' + bdi + ', ' + bda + ', '
+
+        testbed.satisfy_dependencies_string(all_build_deps +
+                                            ', '.join(build_essential),
+                                            arg,
+                                            build_dep=True,
+                                            shell_on_failure=opts.shell_fail)
 
         # keep patches applied for tests
         source_rules_command([dpkg_buildpackage, 'dpkg-source --before-build .'], 'build', cwd=result_pwd)
@@ -733,7 +765,8 @@ def process_actions():
                 (tests, skipped) = testdesc.parse_debian_source(
                     tests_tree.host, testbed.caps, testbed.dpkg_arch,
                     control_path=control_override,
-                    auto_control=opts.auto_control)
+                    auto_control=opts.auto_control,
+                    cross_arch=opts.architecture)
             except testdesc.InvalidControl as e:
                 adtlog.badpkg(str(e))
 
diff --git a/runner/autopkgtest.1 b/runner/autopkgtest.1
index ce4a1bf..abf181b 100644
--- a/runner/autopkgtest.1
+++ b/runner/autopkgtest.1
@@ -136,6 +136,12 @@ autopkgtest --installed-click com.example.myclick -- [...]
 .SH TEST OPTIONS
 
 .TP
+.BR -a " | " --architecture ARCH
+Run tests for the specified architecture, rather than for the host
+architecture as defined by dpkg \-\-print-architecture. When building
+packages from source, cross-build for the target architecture as well.
+
+.TP
 .BR -B " | " --no-built-binaries
 Binaries from unbuilt source packages (see above)
 will not be built or ignored, and dependencies are satisfied with packages from