Merge lp:~mwhudson/ubuntu-archive-scripts/rcbuggy-deployment-fun into lp:ubuntu-archive-scripts

[Michael Hudson-Doyle]

This time for sure? Should use the chdists properly, does not depend on reverse-depends any more. The uninstallability testing might not work because there are no such cases currently...

=== modified file 'find-rcbuggy-problem-packages'
--- find-rcbuggy-problem-packages	2019-09-11 01:33:41 +0000
+++ find-rcbuggy-problem-packages	2019-09-12 03:12:18 +0000
@@ -62,10 +62,13 @@
     cp = subprocess.run(cmd, **kw)
     return cp.stdout.decode(encoding).strip()
 
-def extract_uninstallable_to_src_pkg(output_fp):
-    # Extract a mapping from binary package name to the set of source
-    # packages whose migration from proposed makes that package
-    # uninstallable.
+all_arches = set()
+
+def extract_bin_pkg_arch_to_blocked_src_pkgs(output_fp):
+    # Extract a mapping from binary package name / architectures to
+    # source package the migration of which would make that package
+    # uninstallable (and for convenience, return all architectures
+    # used as keys -- usually there will only be one or two).
 
     # We're looking for sequences of lines like this:
 
@@ -76,11 +79,31 @@
     # (Britney tries to migrate batches of packages but it always
     # tries each package on its own as well).
 
-    r = {}
+    # For each potential migration, britney checks architectures in
+    # sequence and stops when it finds one that regresses (or proceeds
+    # with the migration if it doesn't find one). This means that we
+    # can miss blocking packages here --- for example if migrating $src
+    # would make say $binpkg/amd64 uninstallable, but britney happens
+    # to check arm64 -- where $binpkg does not exist -- and there are
+    # regressions there, we will never find out about the problem
+    # $binpkg causes.  This isn't really too bad because clearly in
+    # this case the migration of $src is blocked by other things that
+    # need to be resolved, but it does mean that packages might appear
+    # and disapper from the report depending on the order that britney
+    # checks architectures in (which is not consistent from run to
+    # run). C'est la vie.
+
+    bin_pkg_arch_to_blocked_src_pkgs = {}
     srcpkg = None
+    arch_prefix = "Arch order is: "
     for line in output_fp:
-        parts = line.split()
+        line = line.strip()
+        if line.startswith(arch_prefix):
+            all_arches.update(line[len(arch_prefix):].split(', '))
+        parts = line.split(None, 2)
         if len(parts) >= 2:
+            if parts[0] in {"Trying", "trying"}:
+                srcpkg = None
             if parts[0] == 'skipped:':
                 srcpkg = None
                 # If parts[2] is '(' then this line is about trying to
@@ -88,13 +111,26 @@
                 # looking for.
                 if parts[2].startswith('('):
                     srcpkg = parts[1]
-            if srcpkg is not None and parts[0] == '*':
+            if srcpkg is not None and srcpkg[0] != '-' and parts[0] == '*':
                 # parts[1] is "${arch}:"
                 # parts[2:] is a comma+space separated list of binary package names.
-                for binpkg in parts[2:]:
-                    binpkg = binpkg.strip(',')
-                    r.setdefault(binpkg, set()).add(srcpkg)
-    return r
+                arch = parts[1][:-1]
+                for binpkg in parts[2].split(', '):
+                    bin_pkg_arch_to_blocked_src_pkgs.setdefault(
+                        (binpkg, arch), set()).add(srcpkg)
+    return bin_pkg_arch_to_blocked_src_pkgs
+
+
+def chdist_grep_dctrl_packages(arch, *args):
+    return run_output(
+        "chdist", "grep-dctrl-packages", "{}-{}".format(series, arch),
+        "-nsPackage", *args, check=False).splitlines()
+
+
+def chdist_grep_dctrl_sources(arch, *args):
+    return run_output(
+        "chdist", "grep-dctrl-sources", "{}-{}".format(series, arch),
+        "-nsPackage", *args, check=False).splitlines()
 
 
 @attr.s
@@ -102,9 +138,66 @@
     source_package_name = attr.ib()
     bugs = attr.ib()
     block_by_regression = attr.ib(default=attr.Factory(set))
-    block_by_uninstallability = attr.ib(default=attr.Factory(list))
+    block_by_uninstallability = attr.ib(default=attr.Factory(set))
     suites = attr.ib(default=attr.Factory(set))
     _rdeps_lines = attr.ib(default=None)
+    _binpkgs = attr.ib(default=None)
+
+    def binary_pkgs(self):
+        if self._binpkgs is None:
+            self._binpkgs = set()
+            for arch in all_arches:
+                arch_binpkgs = chdist_grep_dctrl_packages(
+                    arch, "-wS", self.source_package_name)
+                self._binpkgs.update({(binpkg, arch) for binpkg in arch_binpkgs})
+        return self._binpkgs
+
+    def reverse_depends(self):
+        if self._rdeps_lines is None:
+            # These are maps rdep -> binpkg -> arches
+            reverse_recommends = {}
+            reverse_depends = {}
+            # This just maps rbdep -> binpkgs
+            reverse_build_depends = {}
+            for binpkg, arch in self.binary_pkgs():
+                for rec in chdist_grep_dctrl_packages(
+                        arch, "-wFRecommends", binpkg):
+                    if (rec, arch) in self.binary_pkgs():
+                        continue
+                    reverse_recommends.setdefault(rec, {}).setdefault(binpkg, set()).add(arch)
+                for dep in chdist_grep_dctrl_packages(
+                        arch, "-wFDepends", binpkg):
+                    if (dep, arch) in self.binary_pkgs():
+                        continue
+                    reverse_depends.setdefault(dep, {}).setdefault(binpkg, set()).add(arch)
+                for bdeb in chdist_grep_dctrl_sources(
+                    arch, "-wFBuild-Depends", binpkg, "--or", "-wFBuild-Depends-Indep", binpkg):
+                    reverse_build_depends.setdefault(bdeb, set()).add(binpkg)
+            self._rdeps_lines = []
+            if reverse_depends:
+                self._rdeps_lines.append("Reverse-Depends")
+                for rdep in sorted(reverse_depends):
+                    for binpkg in sorted(reverse_depends[rdep]):
+                        if reverse_depends[rdep][binpkg] == all_arches:
+                            arches = "all architectures"
+                        else:
+                            arches = ", ".join(reverse_depends[rdep][binpkg])
+                        self._rdeps_lines.append(" ".join(["*", rdep, "for", binpkg, "on", arches]))
+            if reverse_recommends:
+                self._rdeps_lines.append("Reverse-Recommends")
+                for rdep in sorted(reverse_recommends):
+                    for binpkg in sorted(reverse_recommends[rdep]):
+                        if reverse_recommends[rdep][binpkg] == all_arches:
+                            arches = "all architectures"
+                        else:
+                            arches = ", ".join(reverse_recommends[rdep][binpkg])
+                        self._rdeps_lines.append(" ".join(["*", rdep, "for", binpkg, "on", arches]))
+            if reverse_build_depends:
+                self._rdeps_lines.append("Reverse-Build-Depends")
+                for rdep in sorted(reverse_build_depends):
+                    for binpkg in sorted(reverse_build_depends[rdep]):
+                        self._rdeps_lines.append(" ".join(["*", rdep, "for", binpkg]))
+        return self._rdeps_lines
 
     @property
     def comment(self):
@@ -119,32 +212,12 @@
         return comment
 
     @property
-    def rdeps_lines(self):
-        if self._rdeps_lines is None:
-            rdeps = run_output("reverse-depends", 'src:' + self.source_package_name)
-            rbdeps = run_output("reverse-depends", '-b', 'src:' + self.source_package_name)
-
-            if rdeps == 'No reverse dependencies found':
-                rdeps = ''
-            if rbdeps == 'No reverse dependencies found':
-                rbdeps = ''
-
-            if rdeps or rbdeps:
-                if rdeps and rbdeps:
-                    self._rdeps_lines = (rdeps + '\n' + rbdeps).splitlines()
-                else:
-                    self._rdeps_lines = (rdeps + rbdeps).splitlines()
-            else:
-                self._rdeps_lines = []
-        return self._rdeps_lines
-
-    @property
     def rdeps_text_short(self):
-        return "\n".join(self.rdeps_lines[:10])
+        return "\n".join(self.reverse_depends()[:10])
 
     @property
     def rdeps_text_more(self):
-        return "\n".join(self.rdeps_lines[10:])
+        return "\n".join(self.reverse_depends()[10:])
 
     @property
     def removal_commands(self):
@@ -154,14 +227,14 @@
         cmds = []
         for suite in suites:
             cmd = " ".join(["remove-package", "-s", suite, "-m", shlex.quote(self.comment), self.source_package_name])
-            if self.rdeps_lines:
+            if self.reverse_depends():
                 cmd = '#' + cmd
             cmds.append(cmd)
         return cmds
 
     @property
     def css_class(self):
-        if self.rdeps_lines:
+        if self.reverse_depends():
             return 'removal-not-ok'
         else:
             return 'removal-ok'
@@ -178,7 +251,7 @@
     with open(args.ubuntu_excuses) as fp:
         ubuntu_excuses = yaml.load(fp, Loader=yaml.CSafeLoader)
     with open(args.ubuntu_update_output) as fp:
-        uninstallable_to_src_pkg = extract_uninstallable_to_src_pkg(fp)
+        bin_pkg_arch_to_blocked_src_pkgs = extract_bin_pkg_arch_to_blocked_src_pkgs(fp)
     with open(args.debian_excuses) as fp:
         debian_excuses = yaml.load(fp, Loader=yaml.CSafeLoader)
 
@@ -219,18 +292,17 @@
             if source['old-version'] != '-':
                 rc_gones[item].suites.add(series)
     print("checking uninstallability")
-    for pkg in rc_gones:
-        if pkg not in in_proposed_by_autopkgtest_or_missing_binaries:
+    for rc_gone in rc_gones.values():
+        if rc_gone.source_package_name not in in_proposed_by_autopkgtest_or_missing_binaries:
             continue
-        bin_pkgs = run_output("chdist", "grep-dctrl-packages", series, "-SwnsPackage", pkg, check=False)
-        for bin_pkg in set(bin_pkgs.splitlines()):
-            rc_gones[pkg].block_by_uninstallability.extend(uninstallable_to_src_pkg.get(bin_pkg, []))
+        for bin_pkg, arch in set(rc_gone.binary_pkgs()):
+            rc_gone.block_by_uninstallability.update(bin_pkg_arch_to_blocked_src_pkgs.get((bin_pkg, arch), set()))
     print("finding reverse-deps")
     packages = []
     for _, rc_gone in sorted(rc_gones.items()):
         if not rc_gone.block_by_regression and not rc_gone.block_by_uninstallability:
             continue
-        rc_gone.rdeps_lines
+        rc_gone.reverse_depends()
         packages.append(rc_gone)
 
     print("rendering")
 
=== modified file 'templates/rcbuggy-problem-packages.html'
--- templates/rcbuggy-problem-packages.html	2019-09-09 01:48:46 +0000
+++ templates/rcbuggy-problem-packages.html	2019-09-12 03:12:18 +0000
@@ -62,10 +62,10 @@
         {% endif %}
         {% if package.block_by_uninstallability %}
         <p>
-          At least one binary from this package is made uninstallable by the versions of {{ package.block_by_uninstallability|join(", ") }} in -proposed.
+          At least one binary from this package is made uninstallable by the versions of {{ package.block_by_uninstallability|sort|join(", ") }} in -proposed.
         </p>
         {% endif %}
-        {% if package.rdeps_lines %}
+        {% if package._rdeps_lines %}
         <pre class="rdeps">
 {{ package.rdeps_text_short }}
 {% if package.rdeps_text_more -%}