Merge lp:~brian-murray/ubuntu-archive-tools/or-deps into lp:ubuntu-archive-tools

[Brian Murray]

This is a resurrection of mterry's branch to OR deps in the NBS report which was originally registered here:

https://code.launchpad.net/~mterry/ubuntu-archive-tools/or-deps/+merge/115445

I had to make some changes to it due to the architecture and recommends changes, well and because it was almost 9 years old, but I think its good to go now.

I've tested nbs-report on snakefruit with the current NBS output for impish and I also ran checkrdepends on a few packages which aren't NBS and used that as input for nbs-report as a further test.

[Brian Murray (brian-murray)]

It specifically needed to be updated in light of the following two changes:

https://bazaar.launchpad.net/~ubuntu-archive/ubuntu-archive-tools/trunk/revision/732
https://bazaar.launchpad.net/~ubuntu-archive/ubuntu-archive-tools/trunk/revision/734

[Steve Langasek (vorlon)]

Currently the NBS report on https://people.canonical.com/~ubuntu-archive/nbs.html is empty so we can't see whether the output differs. How can we test this before merging, to confirm that the output is what we expect with the new feature?

[Brian Murray (brian-murray)]

Looking at cron.NBS we can see that archive-cruft is used to generate a list of package for checkrdepends to run on and then nbs-report uses the output of checkrdepends to create its report. So one could just run checkrdepends for a selection of interesting packages, which are not cruft, and then run nbs-report to confirm the output.

[Steve Langasek (vorlon)]

Tested, the output looks sane/correct on a test package.

review: Approve

=== modified file 'checkrdepends'
--- checkrdepends	2021-04-23 08:44:48 +0000
+++ checkrdepends	2021-07-10 01:03:37 +0000
@@ -36,10 +36,13 @@
 # Cheaper version of deb822.PkgRelation.parse_relations.
 def parse_relation_packages(raw):
     for or_dep in raw.split(','):
+        dep_list = []
         for dep in or_dep.split('|'):
             match = re_dep.match(dep.strip())
             if match:
-                yield match.group(1)
+                dep_list.append(match.group(1))
+        for dep in dep_list:
+            yield dep, dep_list
 
 
 def primary_arches(suite):
@@ -81,8 +84,9 @@
         for field in ('build-depends', 'build-depends-indep'):
             if field not in stanza:
                 continue
-            for depname in parse_relation_packages(stanza[field]):
-                build_deps[depname].add(name)
+            for depname, depwhy in parse_relation_packages(stanza[field]):
+                build_deps.setdefault(depname, set())
+                build_deps[depname].add((name, '|'.join(depwhy)))
     return ret
 
 
@@ -95,14 +99,15 @@
             for field in ('pre-depends', 'depends', 'recommends'):
                 if field not in stanza:
                     continue
-                for depname in parse_relation_packages(stanza[field]):
+                for depname, depwhy in parse_relation_packages(stanza[field]):
                     if depname not in debs:
                         continue
                     # skip dependencies that are built from the same source,
                     # when we're doing a sourceful removal.
                     if name in ignores:
                         continue
-                    deps[depname][name] = (field, stanza['architecture'])
+                    deps[depname][name] = (field, stanza['architecture'],
+                                           '|'.join(depwhy))
     except IOError:
         if not missing_ok:
             raise
@@ -134,12 +139,14 @@
         return ('', '-updates', '-security', '-backports')
 
 
-def render_dep(name, field, arch):
+def render_dep(name, field, arch, why):
     ret = name
     if field == "recommends":
         ret += " (r)"
     if arch == "all":
         ret += " [all]"
+    if why:
+        ret += " %s" % why
     return ret
 
 
@@ -188,8 +195,8 @@
                 if deb in build_deps:
                     print("-- %s%s/%s build deps on %s:" %
                           (opts.suite, pocket, comp, deb), file=out)
-                    for pkg in sorted(build_deps[deb]):
-                        print(pkg, file=out)
+                    for pkg, why in sorted(build_deps[deb]):
+                        print(pkg, why, file=out)
 
                 # binary dependencies
                 for arch in arches:
@@ -221,15 +228,17 @@
                     if deb in deps:
                         print("-- %s%s/%s %s deps on %s:" %
                               (opts.suite, pocket, comp, arch, deb), file=out)
-                        for pkg, (field, pkgarch) in sorted(deps[deb].items()):
-                            print(render_dep(pkg, field, pkgarch), file=out)
+                        for pkg, (field, pkgarch, why) in sorted(deps[deb].items()):
+                            print(render_dep(pkg, field, pkgarch, why),
+                                  file=out)
                     if deb in di_deps:
                         print("-- %s%s/%s %s deps on %s:" %
                               (opts.suite, pocket, di_comp, arch, deb),
                               file=out)
                         for pkg, (field, pkgarch) in sorted(
                                 di_deps[deb].items()):
-                            print(render_dep(pkg, field, pkgarch), file=out)
+                            print(render_dep(pkg, field, pkgarch, why),
+                                  file=out)
 
                 if opts.directory is not None:
                     out.close()

=== modified file 'nbs-report'
--- nbs-report	2021-04-23 08:44:48 +0000
+++ nbs-report	2021-07-10 01:03:37 +0000
@@ -39,6 +39,7 @@
 
     cur_component = None
     cur_arch = None
+    why = None
 
     with open(path) as f:
         for line in f:
@@ -47,9 +48,13 @@
                 continue
             assert cur_component
             assert cur_arch
-
+            # line can contain " (r)" or " [all]" in the middle
             rdep = line.strip().split()[0]
-            pkgmap.setdefault(rdep, (cur_component, []))[1].append(cur_arch)
+            pkgmap.setdefault(rdep, (cur_component, [], []))[1].append(cur_arch)
+            if len(line.strip().split()) >= 2:
+                why = line.strip().split()[-1]
+                if why not in pkgmap[rdep][2]:
+                    pkgmap[rdep][2].append(why)
 
 
 def _pkg_removable(options, pkg, nbs, checked_v):
@@ -115,7 +120,7 @@
             except KeyError:
                 pass
             return False
-        if not _pkg_removable(options, rdep, nbs, checked_v):
+        elif not _pkg_removable(options, rdep, nbs, checked_v):
             try:
                 checked_v.remove(rdep)
             except KeyError:
@@ -198,12 +203,12 @@
             cls = 'removable'
         else:
             cls = 'normal'
-        print('<tr><th colspan="4"><span class="%s">%s</span></th></tr>\n' %
+        print('<tr><td colspan="5"><span class="%s">%s</span></td></tr>\n' %
               (cls, pkg), end="")
         for rdep in sorted(nbsmap):
             if rdep in rdeps_with_alternates:
                 continue
-            (component, arches) = nbsmap[rdep]
+            (component, arches, why) = nbsmap[rdep]
 
             if component in ('main', 'restricted'):
                 component_cls = 'sup'
@@ -220,11 +225,16 @@
                 reverse_nbs[rdep].append(pkg)
                 pkg_component[rdep] = (component, component_cls)
 
+            whyhtml = ''
+            if why != pkg:
+                whyhtml = ' | '.join(why)
+
             print('<tr><td>&nbsp; &nbsp; </td>', end='')
             print('<td><span class="%s">%s</span></td> ' % (cls, rdep), end='')
             print('<td><span class="component%s">%s</span></td>' %
                   (component_cls, component), end='')
-            print('<td>%s</td></tr>' % ' '.join(arches))
+            print('<td>%s</td>' % ' '.join(arches), end='')
+            print('<td>%s</td></tr>' % whyhtml)
 
     print('''</table>
 <h2>Packages which depend on NBS packages</h2>
@@ -276,7 +286,7 @@
 
     options.time = time.time()
 
-    # pkg -> rdep_pkg -> (component, [arch1, arch2, ...])
+    # pkg -> rdep_pkg -> (component, [arch1, arch2, ...], [why])
     nbs = defaultdict(dict)
 
     for f in os.listdir(args[0]):