Merge lp:~cjwatson/launchpad-buildd/complex-build-deps into lp:launchpad-buildd

[Colin Watson]

Handle architecture restrictions, architecture qualifications, and restriction formulas (build profiles) in build-dependencies.

Handle architecture restrictions, architecture qualifications, and restriction formulas (build profiles) in build-dependencies, since now that we're doing our own build-dependency analysis we have to worry about this. Architecture qualifications and restriction formulas require a newer version of python-debian, which is in https://launchpad.net/~cjwatson/+archive/ubuntu/launchpad/+packages.

[William Grant (wgrant)]

review: Approve (code)

[lp:~cjwatson/launchpad-buildd/complex-build-deps]

166. By Colin Watson on 2015-07-28

Sync comment with reality.

167. By Colin Watson on 2015-07-28

Adjust variable names.

168. By Colin Watson on 2015-07-28

Clarify comment about !other-architecture restrictions.

=== modified file 'debian/changelog'
--- debian/changelog	2015-07-16 13:03:35 +0000
+++ debian/changelog	2015-07-28 13:58:14 +0000
@@ -5,6 +5,8 @@
     build-dependencies are uninstallable.
   * If there is a mix of definite and dubious dep-wait output, then analyse
     the situation rather than trusting just the definite information.
+  * Handle architecture restrictions, architecture qualifications, and
+    restriction formulas (build profiles) in build-dependencies.
 
  -- Colin Watson <[e-mail]>  Thu, 16 Jul 2015 14:00:16 +0100
 

=== modified file 'lpbuildd/binarypackage.py'
--- lpbuildd/binarypackage.py	2015-07-16 13:03:35 +0000
+++ lpbuildd/binarypackage.py	2015-07-28 13:58:14 +0000
@@ -6,6 +6,7 @@
 from collections import defaultdict
 import os
 import re
+import subprocess
 import traceback
 
 import apt_pkg
@@ -61,6 +62,25 @@
         }
 
 
+class DpkgArchitectureCache:
+    """Cache the results of asking questions of dpkg-architecture."""
+
+    def __init__(self):
+        self._matches = {}
+
+    def match(self, arch, wildcard):
+        if (arch, wildcard) not in self._matches:
+            command = ["dpkg-architecture", "-i%s" % wildcard]
+            env = dict(os.environ)
+            env["DEB_HOST_ARCH"] = arch
+            ret = (subprocess.call(command, env=env) == 0)
+            self._matches[(arch, wildcard)] = ret
+        return self._matches[(arch, wildcard)]
+
+
+dpkg_architecture = DpkgArchitectureCache()
+
+
 class BinaryPackageBuildState(DebianBuildState):
     SBUILD = "SBUILD"
 
@@ -180,18 +200,42 @@
     def relationMatches(self, dep, available):
         """Return True iff a dependency matches an available package.
 
-        :param dep: A dictionary with at least a "name" key, perhaps also a
-            "version" key, and optionally other keys, of the kind returned
-            in a list of lists by
+        :param dep: A dictionary with at least a "name" key, perhaps also
+            "version", "arch", and "restrictions" keys, and optionally other
+            keys, of the kind returned in a list of lists by
             `debian.deb822.PkgRelation.parse_relations`.
         :param available: A dictionary mapping package names to a list of
             the available versions of each package.
         """
+        dep_arch = dep.get("arch")
+        if dep_arch is not None:
+            arch_match = False
+            for enabled, arch_wildcard in dep_arch:
+                if dpkg_architecture.match(self.arch_tag, arch_wildcard):
+                    arch_match = enabled
+                    break
+                elif not enabled:
+                    # Any !other-architecture restriction implies that this
+                    # architecture is allowed, unless it's specifically
+                    # excluded by some other restriction.
+                    arch_match = True
+            if not arch_match:
+                # This dependency "matches" in the sense that it's ignored
+                # on this architecture.
+                return True
+        dep_restrictions = dep.get("restrictions")
+        if dep_restrictions is not None:
+            if all(
+                any(restriction.enabled for restriction in restrlist)
+                for restrlist in dep_restrictions):
+                # This dependency "matches" in the sense that it's ignored
+                # when no build profiles are enabled.
+                return True
         if dep["name"] not in available:
             return False
-        if dep.get("version") is None:
+        dep_version = dep.get("version")
+        if dep_version is None:
             return True
-        dep_version = dep.get("version")
         operator_map = {
             "<<": (lambda a, b: a < b),
             "<=": (lambda a, b: a <= b),
@@ -230,7 +274,14 @@
             unsat_deps = []
             for or_dep in deps:
                 if not any(self.relationMatches(dep, avail) for dep in or_dep):
-                    unsat_deps.append(or_dep)
+                    stripped_or_dep = []
+                    for simple_dep in or_dep:
+                        stripped_simple_dep = dict(simple_dep)
+                        stripped_simple_dep["arch"] = None
+                        stripped_simple_dep["archqual"] = None
+                        stripped_simple_dep["restrictions"] = None
+                        stripped_or_dep.append(stripped_simple_dep)
+                    unsat_deps.append(stripped_or_dep)
             if unsat_deps:
                 return PkgRelation.str(unsat_deps)
         except Exception:

=== modified file 'lpbuildd/tests/test_binarypackage.py'
--- lpbuildd/tests/test_binarypackage.py	2015-07-16 13:03:35 +0000
+++ lpbuildd/tests/test_binarypackage.py	2015-07-28 13:58:14 +0000
@@ -426,6 +426,48 @@
                     "debhelper (>= 9~), foo (>= 1), bar (<< 1) | bar (>= 2)"),
                 {"debhelper": set(["9"]), "bar": set(["1", "1.5"])}))
 
+    def test_analyseDepWait_strips_arch_restrictions(self):
+        # analyseDepWait removes architecture restrictions (e.g. "[amd64]")
+        # from the unsatisfied build-dependencies it returns, and only
+        # returns those relevant to the current architecture.
+        self.buildmanager.initiate(
+            {'foo_1.dsc': ''}, 'chroot.tar.gz',
+            {'distribution': 'ubuntu', 'suite': 'warty',
+             'ogrecomponent': 'main', 'arch_tag': 'i386'})
+        self.assertEqual(
+            "foo (>= 1)",
+            self.buildmanager.analyseDepWait(
+                PkgRelation.parse_relations(
+                    "foo (>= 1) [any-i386], bar (>= 1) [amd64]"),
+                {}))
+
+    def test_analyseDepWait_strips_arch_qualifications(self):
+        # analyseDepWait removes architecture qualifications (e.g. ":any")
+        # from the unsatisfied build-dependencies it returns.
+        self.buildmanager.initiate(
+            {'foo_1.dsc': ''}, 'chroot.tar.gz',
+            {'distribution': 'ubuntu', 'suite': 'warty',
+             'ogrecomponent': 'main', 'arch_tag': 'i386'})
+        self.assertEqual(
+            "foo",
+            self.buildmanager.analyseDepWait(
+                PkgRelation.parse_relations("foo:any, bar:any"),
+                {"bar": set(["1"])}))
+
+    def test_analyseDepWait_strips_restrictions(self):
+        # analyseDepWait removes restrictions (e.g. "<stage1>") from the
+        # unsatisfied build-dependencies it returns, and only returns those
+        # that evaluate to true when no build profiles are active.
+        self.buildmanager.initiate(
+            {'foo_1.dsc': ''}, 'chroot.tar.gz',
+            {'distribution': 'ubuntu', 'suite': 'warty',
+             'ogrecomponent': 'main', 'arch_tag': 'i386'})
+        self.assertEqual(
+            "foo",
+            self.buildmanager.analyseDepWait(
+                PkgRelation.parse_relations("foo <!nocheck>, bar <stage1>"),
+                {}))
+
     def startDepFail(self, error, dscname=''):
         self.startBuild(dscname=dscname)
         write_file(