Merge lp:~wgrant/lp-ftbfs-report/multi-archive-superseded into lp:lp-ftbfs-report

[William Grant]

Add support for detecting superseded builds in non-primary archives.

This branch adds support for identifying superseded builds in non-primary archives. If the archive is non-primary, the distribution's primary archive is checked for an active publication for each build. If no active publication is found, the build is shown in a separate section.

Also a couple of fixes to be more robust against lazr.restfulclient versions (catching a KeyError when looking up people), and against broken builds (handling an absent build_log_url).

=== modified file 'source/build_status.html'
--- source/build_status.html	2011-10-07 04:09:43 +0000
+++ source/build_status.html	2011-10-07 04:09:43 +0000
@@ -184,6 +184,20 @@
         {{ table(lst, ps, 'Also belongs to another packageset?') }}
     {% endfor -%}
 
+    {% if main_archive %}
+    <h2 id="main_superseded">main (superseded): {{ main_superseded|count }} packages (<a href="#top">top</a>)</h2>
+    {{ table(main_superseded) }}
+
+    <h2 id="restricted_superseded">restricted (superseded): {{ restricted_superseded|count }} packages (<a href="#top">top</a>)</h2>
+    {{ table(restricted_superseded) }}
+
+    <h2 id="universe_superseded">universe (superseded): {{ universe_superseded|count }} packages (<a href="#top">top</a>)</h2>
+    {{ table(universe_superseded) }}
+
+    <h2 id="multiverse_superseded">multiverse (superseded): {{ multiverse_superseded|count }} packages (<a href="#top">top</a>)</h2>
+    {{ table(multiverse_superseded) }}
+    {%- endif %}
+
     <hr/>
     <p style="font-size:smaller">Last update: {{ lastupdate }}</p>
     <p>

=== modified file 'source/build_status.py'
--- source/build_status.py	2011-10-07 04:09:43 +0000
+++ source/build_status.py	2011-10-07 04:09:43 +0000
@@ -56,7 +56,8 @@
                 lp_object = launchpad.load(personteam_link)
                 personteam.display_name = lp_object.display_name
                 personteam.name = lp_object.name
-
+            except KeyError:
+                return None
             except HTTPError, e:
                 if e.response.status == 410:
                     personteam = None
@@ -115,9 +116,11 @@
     def __cmp__(self, other):
         return cmp(self.name, other.name)
 
-    def isFTBFS(self, arch_list = default_arch_list):
+    def isFTBFS(self, arch_list = default_arch_list, current = True):
         ''' Returns True if at least one FTBFS exists. '''
         for ver in self.versions:
+            if ver.current != current:
+                continue
             for arch in arch_list:
                 log = ver.getArch(arch)
                 if log is not None:
@@ -149,11 +152,13 @@
 
             # fill the new SPPH object with data
             lp_object = launchpad.load(spph_link)
+            spph._lp = lp_object
             spph.logs = dict()
             spph.version = lp_object.source_package_version
             spph.pocket = lp_object.pocket
             spph.changed_by = PersonTeam(lp_object.package_creator_link)
             #spph.signed_by = spph._lp.package_signer_link and PersonTeam(lp_object.package_signer_link)
+            spph.current = None
             SourcePackage(lp_object).versions.append(spph)
 
             # add to cache
@@ -178,8 +183,10 @@
 
             if self.buildstate == 'UPLOADFAIL':
                 self.log = translate_api_web(build.upload_log_url)
-            else:
+            elif build.build_log_url:
                 self.log = translate_api_web(build.build_log_url)
+            else:
+                self.log = ''
 
             if self.buildstate == 'MANUALDEPWAIT':
                 self.tooltip = 'waits on %s' % build.dependencies
@@ -201,14 +208,14 @@
         return u'Changed-By: %s' % (self.changed_by)
 
 
-def fetch_pkg_list(archive, series, state, last_published, arch_list=default_arch_list):
+def fetch_pkg_list(archive, series, state, last_published, arch_list=default_arch_list, main_archive=None, main_series=None):
     print "Processing '%s'" % state
 
     cur_last_published = None
     # XXX wgrant 2009-09-19: This is an awful hack. We should really
     # just let IArchive.getBuildRecords take a series argument.
     if archive.name == 'primary':
-        buildlist = series.getBuildRecords(build_state = state, pocket='Proposed')
+        buildlist = series.getBuildRecords(build_state = state)
     else:
         buildlist = archive.getBuildRecords(build_state = state)
 
@@ -232,29 +239,52 @@
 
         print "  %s %s" % (build.datebuilt, build.title)
 
+        spph = SPPH(csp_link)
+
+        if spph.current is None:
+            # If a main archive is specified, we check if the current source
+            # is still published there. If it isn't, then it's out of date.
+            # We should make this obvious.
+            # The main archive will normally be the primary archive, and
+            # probably only makes sense if the target archive is a rebuild.
+            if main_archive:
+                main_publications = main_archive.getPublishedSources(
+                    distro_series=main_series,
+                    exact_match=True,
+                    source_name=spph._lp.source_package_name,
+                    version=spph._lp.source_package_version,
+                    status='Published')
+                spph.current = len(main_publications[:1]) > 0
+            else:
+                spph.current = True
+
+        if not spph.current:
+            print "    superseded"
         SPPH(csp_link).addBuildLog(build)
 
     return cur_last_published
 
 
-def generate_page(archive, series, archs_by_archive, template = 'build_status.html', arch_list = default_arch_list):
+def generate_page(archive, series, archs_by_archive, main_archive, template = 'build_status.html', arch_list = default_arch_list):
     try:
         out = open('../%s-%s.html' % (archive.name, series.name), 'w')
     except IOError:
         return
 
     # sort the package lists
-    filter_ftbfs = lambda pkglist: filter(methodcaller('isFTBFS', arch_list), sorted(pkglist))
+    filter_ftbfs = lambda pkglist, current: filter(methodcaller('isFTBFS', arch_list, current), sorted(pkglist))
     data = {}
     for comp in ('main', 'restricted', 'universe', 'multiverse'):
-        data[comp] = filter_ftbfs(components[comp])
+        data[comp] = filter_ftbfs(components[comp], True)
+        data['%s_superseded' % comp] = filter_ftbfs(components[comp], False)
     for pkgset, pkglist in packagesets_ftbfs.items():
-        packagesets_ftbfs[pkgset] = filter_ftbfs(pkglist)
+        packagesets_ftbfs[pkgset] = filter_ftbfs(pkglist, True)
 
     # container object to hold the counts and the tooltip
     class StatData(object):
-        def __init__(self, cnt, tooltip):
+        def __init__(self, cnt, cnt_superseded, tooltip):
             self.cnt = cnt
+            self.cnt_superseded = cnt_superseded
             self.tooltip = tooltip
 
     # compute some statistics (number of packages for each build failure type)
@@ -264,21 +294,25 @@
         for arch in arch_list:
             tooltip = []
             cnt = 0
+            cnt_sup = 0
             for comp in ('main', 'restricted', 'universe', 'multiverse'):
                 s = sum([pkg.getCount(arch, state) for pkg in data[comp]])
-                if s:
+                s_sup = sum([pkg.getCount(arch, state) for pkg in data['%s_superseded' % comp]])
+                if s or s_sup:
                     cnt += s
-                    tooltip.append('<td>%s:</td><td style="text-align:right;">%i</td>' % (comp, s))
+                    cnt_sup += s_sup
+                    tooltip.append('<td>%s:</td><td style="text-align:right;">%i (%i superseded)</td>' % (comp, s, s_sup))
             if cnt:
                 tooltiphtml = u'<table><tr>'
                 tooltiphtml += u'</tr><tr>'.join(tooltip)
                 tooltiphtml += u'</tr></table>'
-                stats[state][arch] = StatData(cnt, tooltiphtml)
+                stats[state][arch] = StatData(cnt, cnt_sup, tooltiphtml)
             else:
-                stats[state][arch] = StatData(None, None)
+                stats[state][arch] = StatData(None, None, None)
 
     data['stats'] = stats
     data['archive'] = archive
+    data['main_archive'] = main_archive
     data['series'] = series
     data['arch_list'] = arch_list
     data['archs_by_archive'] = archs_by_archive
@@ -338,10 +372,9 @@
 
 if __name__ == '__main__':
     # login anonymously to LP
-    launchpad = Launchpad.login_anonymously('qa-ftbfs', lp_service)
+    launchpad = Launchpad.login_anonymously('qa-ftbfs', lp_service, version=api_version)
 
     ubuntu = launchpad.distributions['ubuntu']
-
     assert len(sys.argv) >= 4
 
     try:
@@ -355,6 +388,12 @@
         print 'Error: %s is not a valid series.' % sys.argv[2]
         sys.exit(1)
 
+    if archive.name != 'primary':
+        main_archive = ubuntu.main_archive
+        main_series = ubuntu.current_series
+    else:
+        main_archive = main_series = None
+
     archs_by_archive = dict(main=[], ports=[])
     for arch in sys.argv[3:]:
         das = series.getDistroArchSeries(archtag=arch)
@@ -388,11 +427,11 @@
                 packagesets_ftbfs[ps.name] = [] # empty list to add FTBFS for each package set later
 
         for state in ('Failed to build', 'Dependency wait', 'Chroot problem', 'Failed to upload'):
-            last_published[state] = fetch_pkg_list(archive, series, state, last_published[state], default_arch_list)
+            last_published[state] = fetch_pkg_list(archive, series, state, last_published[state], default_arch_list, main_archive, main_series)
 
         save_timestamps(archive, series, last_published)
 
         print "Generating HTML page..."
-        generate_page(archive, series, archs_by_archive)
+        generate_page(archive, series, archs_by_archive, main_archive)
         print "Generating CSV file..."
         generate_csvfile(archive, series)