Merge ~vorlon/germinate:feature-no-follow-build-depends-all into germinate:master

[Steve Langasek]

I want to be able to use germinate to calculate the packageset for a minimum self-hosting i386 port based on the compatibility libraries we want to support. That needs to include build-dependencies, but only the build-dependencies of packages that we actually build on i386 - which does not include Arch: all packages which themselves satisfy (build-)dependencies for packages in our set.

This MP proposes a new feature that lets us truncate on build-dependencies of arch: all binary packages.

[Colin Watson (cjwatson)]

In addition to the inline comments below, the new feature flags need to be documented in man/germinate.1. When writing this review I noticed that the previously-added (no-)follow-build-depends flags had not been properly documented there, so I did so; you'll thus want to rebase this on master before doing anything else, as that will provide a clearer structure in the documentation into which you can slot in descriptions of the new flags.

review: Needs Fixing

8b3e547... by Steve Langasek on 2019-11-20

Document no-follow-build-depends-all in the manpage.

7b7c007... by Steve Langasek on 2019-11-21

Test case for feature follow-build-depends-all.

Unmerged commits

8b3e547... by Steve Langasek on 2019-11-20

Document no-follow-build-depends-all in the manpage.

ad17b5d... by Steve Langasek on 2019-11-11

Add a new feature, follow-build-depends-all.

This tells whether to follow build-dependencies of packages which are
Architecture: all. This is useful for germinating packagesets for partial
architectures, where the architecture-all binaries will never be built on
that arch but instead of some primary arch, so we should truncate traversal
of build-dependencies to exclude packages we don't need built on the partial
arch.

7b7c007... by Steve Langasek on 2019-11-21

Test case for feature follow-build-depends-all.

diff --git a/germinate/germinator.py b/germinate/germinator.py
index 25911a3..9b4e340 100644
--- a/germinate/germinator.py
+++ b/germinate/germinator.py
@@ -465,6 +465,9 @@ class Germinator(object):
 
         self._packages[pkg]["Essential"] = section.get("Essential", "")
 
+        self._packages[pkg]["Architecture"] = \
+            section.get("Architecture", self._arch)
+
         for field in "Pre-Depends", "Depends", "Recommends", "Built-Using":
             value = section.get(field, "")
             try:
@@ -1252,6 +1255,20 @@ class Germinator(object):
                 return False
         return "no-follow-build-depends" not in structure.features
 
+    def _follow_build_depends_all(self, structure, seed=None):
+        """
+        Test whether Build-Depends should be followed for Architecture: all
+        packages (not required for the special case of partial
+        architectures)
+        Defaults to True if not explicitly specified.
+        """
+        if seed is not None:
+            if "follow-build-depends-all" in seed._features:
+                return True
+            if "no-follow-build-depends-all" in seed._features:
+                return False
+        return "no-follow-build-depends-all" not in structure.features
+
     def _add_reverse(self, pkg, field, rdep):
         """Add a reverse dependency entry."""
         if "Reverse-Depends" not in self._packages[pkg]:
@@ -1642,6 +1659,12 @@ class Germinator(object):
             output._all_srcs.add(pkg_src)
             seed._build_srcs.add(pkg_src)
 
+            # Special-case not following build dependencies of arch: all
+            # packages
+            if not self._follow_build_depends_all(seed.structure, seed) and \
+               self._packages[pkg]["Architecture"] == "all":
+                continue
+
             if self._follow_build_depends(seed.structure, seed):
                 for build_depends in BUILD_DEPENDS:
                     self._add_dependency_tree(
diff --git a/germinate/tests/test_germinator.py b/germinate/tests/test_germinator.py
index 5b84c63..aaab106 100644
--- a/germinate/tests/test_germinator.py
+++ b/germinate/tests/test_germinator.py
@@ -177,6 +177,7 @@ class TestGerminator(TestCase):
             "Provides": [],
             "Kernel-Version": "",
             "Multi-Arch": "none",
+            "Architecture": "i386",
             }, germinator._packages["hello"])
         self.assertEqual("deb", germinator._packagetype["hello"])
         self.assertIn("hello-dependency", germinator._packages)
@@ -195,6 +196,7 @@ class TestGerminator(TestCase):
             "Provides": [],
             "Kernel-Version": "",
             "Multi-Arch": "foreign",
+            "Architecture": "i386",
             }, germinator._packages["hello-dependency"])
         self.assertEqual("deb", germinator._packagetype["hello-dependency"])
         self.assertEqual({}, germinator._provides)
@@ -321,6 +323,44 @@ class TestGerminator(TestCase):
             shutil.rmtree(self.archive_dir)
             shutil.rmtree(self.seeds_dir)
 
+    def test_build_depends_all(self):
+        """Confirm build-depends of arch: all packages are recursed when
+        follow-build-depends-all is set, and not when
+        no-follow-build-depends-all is set.
+        """
+        self.addSource("focal", "main", "hello", "1.0-1", ["hello"],
+                       fields={"Build-Depends": "gettext"})
+        self.addPackage("focal", "main", "i386", "hello", "1.0-1",
+                        fields={"Architecture": "all"})
+        self.addSource("focal", "main", "gettext", "0.18.1.1-5ubuntu3",
+                       ["gettext"])
+        self.addPackage("focal", "main", "i386", "gettext",
+                        "0.18.1.1-5ubuntu3")
+        branch = "collection.focal"
+        archive = TagFile(
+            "focal", "main", "i386", "file://%s" % self.archive_dir)
+        for sense in ("no-", ""):
+            self.addSeed(branch, "base")
+            self.addSeedPackage(branch, "base", "hello")
+            self.addStructureLine(branch,
+                                  "feature %sfollow-build-depends-all" % sense)
+            germinator = Germinator("i386")
+            germinator.parse_archive(archive)
+            structure = self.openSeedStructure(branch)
+            germinator.plant_seeds(structure)
+            germinator.grow(structure)
+
+            expected = set()
+            if sense == "":
+                expected.add("gettext")
+            self.assertEqual(
+                expected, germinator.get_build_depends(structure, "base"),
+                "Build-Depends: gettext from Architecture: all package "
+                "incorrectly %s" % "included" if sense == "no-" else "omitted")
+
+            shutil.rmtree(self.seeds_dir)
+        shutil.rmtree(self.archive_dir)
+
     def test_build_depends_profiles(self):
         """Test that https://wiki.debian.org/BuildProfileSpec restrictions
         are parseable.
diff --git a/man/germinate.1 b/man/germinate.1
index 9f80f03..cd4e48f 100644
--- a/man/germinate.1
+++ b/man/germinate.1
@@ -168,10 +168,18 @@ The following flags are currently defined:
 Follow Build-Depends fields.
 This flag is only recognised in individual seed files, not in
 .Pa STRUCTURE .
+.It follow\-build\-depends\-all
+Follow Build-Depends fields for Architecture: all packages.  This has no
+effect if
+.Pa no\-follow\-build\-depends
+is set.
 .It follow\-recommends
 Treat Recommends fields as if they were Depends.
 .It no\-follow\-build\-depends
 Do not follow Build-Depends fields.
+.It no\-follow\-build\-depends\-all
+Do not follow Build-Depends fields for Architecture: all packages, even
+though Build-Depends are followed for other packages.
 .It no\-follow\-recommends
 Do not treat Recommends fields as if they were Depends.
 This flag is only recognised in individual seed files, not in