Merge lp:~brian-murray/ubuntu-archive-tools/nbs-alt-deps into lp:ubuntu-archive-tools

[Brian Murray]

This modifies the NBS report so that it is aware of packages which have a dependency on a package which is NBS but that dependency also has an alternate (ORed) dependency that is satisfied by another package in the archive.

This is done by reading the Sources.gz and Packages.gz files again to see if the dependency has an alternate one and checking to see if the alternate is also NBS.

The read_tag_file function was moved out of checkrdepends and into utils.py and it is now utilized by nbs-report. The cron.NBS file was also modified so that the archive base directory is passed to nbs-report.

[Brian Murray (brian-murray)]

The output of the updated report can be found here:

https://people.canonical.com/~brian/tmp/nbs.html

[Steve Langasek (vorlon)]

review: Needs Information

[lp:~brian-murray/ubuntu-archive-tools/nbs-alt-deps]

1231. By Brian Murray on 2019-08-19

make changes based on reviewer feedback

[Brian Murray (brian-murray)]

I've answered your questions and made a change or two to the code.

[Steve Langasek (vorlon)]

review: Approve

=== modified file 'checkrdepends'
--- checkrdepends	2019-04-20 00:28:29 +0000
+++ checkrdepends	2019-08-19 23:21:28 +0000
@@ -17,14 +17,12 @@
 from __future__ import print_function
 
 from collections import defaultdict
-import gzip
 import optparse
 import os
 import re
 import sys
-import tempfile
 
-import apt_pkg
+from utils import read_tag_file
 
 
 default_base = '/home/ubuntu-archive/mirror/ubuntu'
@@ -65,34 +63,6 @@
         return ('arm64', 'armhf', 'ppc64el', 's390x')
 
 
-def read_tag_file(path):
-    tmp = tempfile.NamedTemporaryFile(prefix='checkrdepends.', delete=False)
-    try:
-        compressed = gzip.open(path)
-        try:
-            tmp.write(compressed.read())
-        finally:
-            compressed.close()
-        tmp.close()
-        with open(tmp.name) as uncompressed:
-            tag_file = apt_pkg.TagFile(uncompressed)
-            prev_name = None
-            prev_stanza = None
-            for stanza in tag_file:
-                try:
-                    name = stanza['package']
-                except KeyError:
-                    continue
-                if name != prev_name and prev_stanza is not None:
-                    yield prev_stanza
-                prev_name = name
-                prev_stanza = stanza
-            if prev_stanza is not None:
-                yield prev_stanza
-    finally:
-        os.unlink(tmp.name)
-
-
 def read_sources(path):
     ret = {
         'binary': {},

=== modified file 'cron.NBS'
--- cron.NBS	2019-04-20 00:28:29 +0000
+++ cron.NBS	2019-08-19 23:21:28 +0000
@@ -42,6 +42,6 @@
 
 rsync -a --delete "$D/" "$OUTDIR/"
 
-nbs-report -d "$DISTRIBUTION" -s "$RELEASE" --csv "${OUTFILE%.html}.csv" \
+nbs-report -B "$MIRROR/$DISTRIBUTION" -d "$DISTRIBUTION" -s "$RELEASE" --csv "${OUTFILE%.html}.csv" \
     "$OUTDIR/" >"$OUTFILE.new" && \
     mv "$OUTFILE.new" "$OUTFILE"

=== modified file 'nbs-report'
--- nbs-report	2019-04-20 00:28:29 +0000
+++ nbs-report	2019-08-19 23:21:28 +0000
@@ -2,6 +2,7 @@
 
 # Copyright (C) 2011, 2012  Canonical Ltd.
 # Author: Martin Pitt <[e-mail]>
+# Author: Brian Murray <[e-mail]>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -28,6 +29,10 @@
 import time
 
 from charts import make_chart, make_chart_header
+from utils import read_tag_file
+
+default_base = '/home/ubuntu-archive/mirror/ubuntu'
+rdeps_with_alternates = []
 
 
 def parse_checkrdepends_file(path, pkgmap):
@@ -48,8 +53,8 @@
             pkgmap.setdefault(rdep, (cur_component, []))[1].append(cur_arch)
 
 
-def _pkg_removable(pkg, nbs, checked_v):
-    '''Recursively check if pakcage is removable.
+def _pkg_removable(options, pkg, nbs, checked_v):
+    '''Recursively check if package is removable.
 
     checked_v is the working set of already checked vertices, to avoid infinite
     loops.
@@ -58,14 +63,56 @@
     for rdep in nbs.get(pkg, []):
         if rdep in checked_v:
             continue
-        #checked_v.add(rdep)
-        if not rdep in nbs:
+        global rdeps_with_alternates
+        # utilize a copy of the arches as nbs will be modified
+        arches = list(nbs[pkg][rdep][1])
+        for arch in arches:
+            alternate_available = False
+            if arch == 'build':
+                ptype = 'source'
+                file = 'Sources'
+            else:
+                ptype = 'binary-%s' % arch
+                file = 'Packages'
+            stanzas = read_tag_file('%s/dists/%s/%s/%s/%s.gz' %
+                                    (options.archive_base, options.suite,
+                                     nbs[pkg][rdep][0], ptype, file), rdep)
+            for stanza in stanzas:
+                if 'binary' in ptype:
+                    fields = ('Pre-Depends', 'Depends', 'Recommends')
+                else:
+                    fields = ('Build-Depends', 'Build-Depends-Indep')
+                for field in fields:
+                    if field not in stanza:
+                        continue
+                    if '|' not in stanza[field]:
+                        continue
+                    for or_dep in stanza[field].split(','):
+                        if '|' not in or_dep:
+                            continue
+                        alternatives = [dep.strip()
+                                        for dep in or_dep.split('|')]
+                        if pkg not in alternatives:
+                            continue
+                        for dep in alternatives:
+                            if dep == pkg:
+                                continue
+                            if dep not in nbs:
+                                alternate_available = True
+                                break
+            if alternate_available:
+                nbs[pkg][rdep][1].remove(arch)
+
+        if len(nbs[pkg][rdep][1]) == 0:
+            rdeps_with_alternates.append(rdep)
+
+        if rdep not in nbs and rdep not in rdeps_with_alternates:
             try:
                 checked_v.remove(rdep)
             except KeyError:
                 pass
             return False
-        if not _pkg_removable(rdep, nbs, checked_v):
+        if not _pkg_removable(options, rdep, nbs, checked_v):
             try:
                 checked_v.remove(rdep)
             except KeyError:
@@ -74,7 +121,7 @@
     return True
 
 
-def get_removables(nbs):
+def get_removables(options, nbs):
     '''Get set of removable packages.
 
     This includes packages with no rdepends and disconnected subgraphs, i. e.
@@ -86,10 +133,9 @@
         if p in removable:
             continue
         checked_v = set()
-        if _pkg_removable(p, nbs, checked_v):
-            # we can add the entire cluster here, not just p; avoids
-            # re-checking the other vertices in that cluster
-            removable.update(checked_v)
+        if _pkg_removable(options, p, nbs, checked_v):
+            # we only add packages which are nbs to removable
+            removable.update([p for p in checked_v if p in nbs])
 
     return removable
 
@@ -97,6 +143,7 @@
 def html_report(options, nbs, removables):
     '''Generate HTML report from NBS map.'''
 
+    global rdeps_with_alternates
     print('''\
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
  "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -148,9 +195,11 @@
             cls = 'removable'
         else:
             cls = 'normal'
-        print('<tr><th colspan="4"><span class="%s">%s</span></td></tr>' %
+        print('<tr><th colspan="4"><span class="%s">%s</span></th></tr>\n' %
               (cls, pkg), end="")
         for rdep in sorted(nbsmap):
+            if rdep in rdeps_with_alternates:
+                continue
             (component, arches) = nbsmap[rdep]
 
             if component in ('main', 'restricted'):
@@ -208,6 +257,10 @@
     parser = OptionParser(
         usage="%prog <checkrdepends output directory>",
         description="Generate an HTML report of current NBS binary packages.")
+    parser.add_option('-B', '--archive-base', dest='archive_base',
+                      help=('archive base directory (default: %s)' %
+                            default_base),
+                      default=default_base)
     parser.add_option('-d', '--distribution', default='ubuntu')
     parser.add_option('-s', '--suite', default='eoan')
     parser.add_option(
@@ -246,7 +299,7 @@
     #                    rd.replace('-', '').replace('.', '')), file=dot)
     #    print('}', file=dot)
 
-    removables = get_removables(nbs)
+    removables = get_removables(options, nbs)
 
     html_report(options, nbs, removables)
 

=== added file 'utils.py'
--- utils.py	1970-01-01 00:00:00 +0000
+++ utils.py	2019-08-19 23:21:28 +0000
@@ -0,0 +1,55 @@
+#!/usr/bin/python
+
+# Copyright (C) 2019 Canonical Ltd.
+# Author: Brian Murray <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Portions of archive related code that is re-used by various tools."""
+
+import gzip
+import os
+import tempfile
+
+import apt_pkg
+
+
+def read_tag_file(path, pkg=None):
+    tmp = tempfile.NamedTemporaryFile(prefix='checkrdepends.', delete=False)
+    try:
+        compressed = gzip.open(path)
+        try:
+            tmp.write(compressed.read())
+        finally:
+            compressed.close()
+        tmp.close()
+        with open(tmp.name) as uncompressed:
+            tag_file = apt_pkg.TagFile(uncompressed)
+            prev_name = None
+            prev_stanza = None
+            for stanza in tag_file:
+                try:
+                    name = stanza['package']
+                except KeyError:
+                    continue
+                if pkg:
+                    if name != pkg:
+                        continue
+                if name != prev_name and prev_stanza is not None:
+                    yield prev_stanza
+                prev_name = name
+                prev_stanza = stanza
+            if prev_stanza is not None:
+                yield prev_stanza
+    finally:
+        os.unlink(tmp.name)