Merge lp:~brian-murray/ubuntu-archive-tools/autopkg-excuses into lp:ubuntu-archive-tools

[Brian Murray]

This adds checking of update_excuses.yaml to discover regressions in autopkg test failures and adds information to the report similar to how build failures are displayed. As an example see:

http://people.canonical.com/~brian/tmp/sru-report.html

and the package haproxy, webkitgtk, etc....

The sample report has some formatting issues which I've fixed in the code, I just don't want to bother running the sample report again.

[lp:~brian-murray/ubuntu-archive-tools/autopkg-excuses]

938. By Brian Murray on 2015-02-24

remove code used for testing with one release

939. By Brian Murray on 2015-02-24

sru-report: precise is now being tested and also check for in progress tests.

[Colin Watson (cjwatson)]

review: Approve

=== modified file 'sru-report'
--- sru-report	2015-02-20 17:42:16 +0000
+++ sru-report	2015-02-24 21:04:53 +0000
@@ -37,6 +37,7 @@
     from urllib.request import urlopen
 except ImportError:
     from urllib import urlopen
+import yaml
 
 import apt_pkg
 from launchpadlib.errors import HTTPError
@@ -68,6 +69,8 @@
 releases = {}  # name -> distro_series
 broken_bugs = set()
 ignored_commenters = []
+excuses_url = ("http://people.canonical.com/~ubuntu-archive/proposed-migration"
+               "/%s/update_excuses.yaml")
 
 
 def current_versions(distro_series, sourcename):
@@ -256,8 +259,15 @@
             if builds:
                 builds = '<span style="font-size: x-small">%s</span>' % builds
 
-            print('  <tr><td><a href="%s">%s</a>%s</td> ' %
-                  (lpurl, pkg, builds))
+            autopkg_fails = ''
+            for excuse in rpkg['autopkg_fails']:
+                autopkg_fails += '<br/>%s' % excuse
+            if autopkg_fails:
+                autopkg_fails = '<span style="font-size: x-small">%s</span>' \
+                    % autopkg_fails
+
+            print('  <tr><td><a href="%s">%s</a>%s %s</td> ' %
+                  (lpurl, pkg, builds, autopkg_fails))
             print('  <td><a href="%s">%s</a></td> ' %
                   (lpurl + rpkg['release'], rpkg['release']))
             print('  <td><a href="%s">%s</a></td> ' %
@@ -567,6 +577,7 @@
             'bugs': [buglist],
             'changesfiles': [changes_urls],
             'build_problems': { arch -> (state, URL) },
+            'autopkg_fails': [excuses]
             }
     '''
     srus = defaultdict(dict)
@@ -575,6 +586,16 @@
         #if releases[release].status not in (
         #    "Active Development", "Pre-release Freeze"):
         #    continue # for quick testing
+        pkg_excuses = []
+        if release != 'lucid':
+            excuses_page = excuses_url % release
+            excuses = urlopen(excuses_page)
+            excuses_data = yaml.load(excuses)
+            pkg_excuses = [excuse['source']
+                           for excuse in excuses_data['sources']
+                           if 'autopkgtest' in excuse['reason']
+                           or 'block' in excuse['reason']]
+
         for published in archive.getPublishedSources(
                 pocket='Proposed', status='Published',
                 distro_series=releases[release]):
@@ -605,14 +626,24 @@
             try:
                 for build in published.getBuilds():
                     if not build.buildstate.startswith('Success'):
-                        srus[release][pkg]['build_problems'][build.arch_tag] = (
-                            build.buildstate, build.web_link)
+                        srus[release][pkg]['build_problems'][build.arch_tag] \
+                            = (build.buildstate, build.web_link)
             except HTTPError as e:
                 if e.response['status'] == '401':
                     continue
                 else:
                     raise e
 
+            srus[release][pkg]['autopkg_fails'] = []
+            if pkg in pkg_excuses:
+                for excuse in excuses_data['sources']:
+                    if excuse['source'] == pkg:
+                        for line in excuse['excuses']:
+                            if 'Regression' not in line and \
+                                    'in progress' not in line:
+                                continue
+                            srus[release][pkg]['autopkg_fails'].append(line)
+
     return srus