Merge lp:~brian-murray/ubuntu-archive-tools/sru-remove into lp:ubuntu-archive-tools

[Brian Murray]

This makes it easier to remove packages from -proposed that have failed to be verified in a timely fashion.

It includes a wrapper sru-remove that calls remove-package with a stock message, and updates the SRU bug reports by setting the task to "Won't Fix", removing the verification-needed and / or verification-needed-$series tags, unsubscribing ubuntu-sru and sru-verifcation, and commenting on the bug report. Here is an example of the work it did:

https://bugs.staging.launchpad.net/ubuntu/precise/+source/haproxy/+bug/1038139

It also includes a change to sru-report to include the packages that need to be removed due to a failure to verify e.g.:

sru-remove -s precise -p haproxy 1038139

Feel free to test it using these removable packages:

sru-remove -s precise -p charm-tools 1182905
sru-remove -s precise -p haproxy 1038139
sru-remove -s trusty -p haproxy 1038139
sru-remove -s trusty -p seabios 1366868

=== added file 'sru-remove'
--- sru-remove	1970-01-01 00:00:00 +0000
+++ sru-remove	2015-02-20 17:49:51 +0000
@@ -0,0 +1,159 @@
+#!/usr/bin/python
+
+# Copyright (C) 2015 Brian Murray <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+'''Remove an SRU fom the -proposed pocket for a release.
+
+Remove a package from the -proposed pocket for a release of Ubuntu and comment
+on bug reports regarding the removal of the package giving an explanation that
+it was removed to due a failure for the SRU bug(s) to be verified in a timely
+fashion.
+
+USAGE:
+    sru-remove -s trusty -p homerun 12345
+'''
+
+from __future__ import print_function
+
+import optparse
+import re
+import subprocess
+import sys
+
+from launchpadlib.launchpad import Launchpad
+
+
+def parse_options():
+    '''Parse command line arguments.
+
+    Return (options, [bugs]) tuple.
+    '''
+
+    parser = optparse.OptionParser(
+        usage='Usage: %prog [options] source_package')
+    parser.add_option(
+        "-l", "--launchpad", dest="launchpad_instance", default="production")
+    parser.add_option(
+        "-s", dest="release", default=default_release, metavar="RELEASE",
+        help="release (default: %s)" % default_release)
+    parser.add_option(
+        "-p", "--package", dest="sourcepkg")
+
+    opts, args = parser.parse_args()
+
+    return (opts, args)
+
+
+def process_bug(launchpad, distroseries, sourcepkg, num):
+    bug_target_re = re.compile(
+        r'/ubuntu/(?:(?P<suite>[^/]+)/)?\+source/(?P<source>[^/]+)$')
+    bug = launchpad.bugs[num]
+    series_name = distroseries.name
+    open_task = False
+
+    for task in bug.bug_tasks:
+        # Ugly; we have to do URL-parsing to figure this out.
+        # /ubuntu/+source/foo can be fed to launchpad.load() to get a
+        # distribution_source_package, but /ubuntu/hardy/+source/foo can't.
+        match = bug_target_re.search(task.target.self_link)
+        if (not match or
+            (sourcepkg and
+             match.group('source') != sourcepkg)):
+            print("Ignoring task %s in bug %s" % (task.web_link, num))
+            continue
+        if (match.group('suite') != series_name and
+                match.group('suite') in supported_series and
+                task.status == "Fix Committed"):
+            open_task = True
+        if (match.group('suite') == series_name and
+                task.status == "Fix Committed"):
+            task.status = "Won't Fix"
+            task.lp_save()
+            print("Success: task %s in bug %s" % (task.web_link, num))
+        btags = bug.tags
+        series_v_needed = 'verification-needed-%s' % series_name
+        if series_v_needed in btags:
+            # this dance is needed due to
+            # https://bugs.launchpad.net/launchpadlib/+bug/254901
+            tags = btags
+            tags.remove(series_v_needed)
+            bug.tags = tags
+            bug.lp_save()
+
+        text = ('The version of %s in the proposed pocket of %s that was '
+                'purported to fix this bug report has been removed because '
+                'the bugs that were to be fixed by the upload were not '
+                'verified in a timely (105 days) fashion.' %
+                (sourcepkg, series_name.title()))
+        bug.newMessage(content=text,
+                       subject='Proposed package removed from archive')
+
+    # remove verification-needed tag if there are no open tasks
+    if open_task:
+        return
+    # only unsubscribe the teams if there are no open tasks left
+    bug.unsubscribe(person=launchpad.people['ubuntu-sru'])
+    bug.unsubscribe(person=launchpad.people['sru-verification'])
+    if 'verification-needed' in btags:
+        # this dance is needed due to
+        # https://bugs.launchpad.net/launchpadlib/+bug/254901
+        tags = btags
+        tags.remove('verification-needed')
+        bug.tags = tags
+        bug.lp_save()
+
+
+if __name__ == '__main__':
+
+    default_release = 'utopic'
+    removal_comment = ('The package was removed due to its SRU bug(s) '
+                       'not being verified in a timely fashion.')
+
+    (opts, bugs) = parse_options()
+
+    launchpad = Launchpad.login_with('sru-remove', opts.launchpad_instance,
+                                     version="devel")
+    ubuntu = launchpad.distributions['ubuntu']
+    # determine series for which we issue SRUs
+    supported_series = []
+    for serie in ubuntu.series:
+        if serie.supported:
+            supported_series.append(serie.name)
+
+    series = ubuntu.getSeries(name_or_version=opts.release)
+    archive = ubuntu.main_archive
+
+    existing = [
+        pkg for pkg in archive.getPublishedSources(
+            exact_match=True, distro_series=series, pocket='Proposed',
+            source_name=opts.sourcepkg, status='Published')]
+
+    if not existing:
+        print("ERROR: %s was not found in -proposed for release %s." %
+              (opts.sourcepkg, opts.release), file=sys.stderr)
+        sys.exit(1)
+
+    rm_p_cmd = ["remove-package", "-m", removal_comment, "-y",
+                "-l", opts.launchpad_instance, "-s",
+                "%s-proposed" % opts.release, opts.sourcepkg]
+    ret = subprocess.call(rm_p_cmd)
+    if ret != 0:
+        print("ERROR: There was an error removing %s from %s-proposed.\n"
+              "The remove-package command returned %s." %
+              (opts.sourcepkg, opts.release, ret), file=sys.stderr)
+        sys.exit(1)
+    # only comment on the bugs after removing the package
+    for bug in bugs:
+        process_bug(launchpad, series, opts.sourcepkg, bug)

=== modified file 'sru-report'
--- sru-report	2014-10-01 22:31:35 +0000
+++ sru-report	2015-02-20 17:49:51 +0000
@@ -210,6 +210,8 @@
     pkgcleanup = []
     pkgcleanup_release = []
     pkgsuperseded = []
+    # set of (series_name, srcpkg, [bugs])
+    proposed_antique = []
     for release in sorted(srus):
         if not srus[release]:
             continue
@@ -263,6 +265,8 @@
             print('  <td><a href="%s">%s</a></td> ' %
                   (lpurl + rpkg['proposed'], rpkg['proposed']))
             print('  <td>')
+            removable = True
+            antique = False
             for b, t in sorted(rpkg['bugs'].iteritems()):
                 cls = ' class="'
                 hover_text = ''
@@ -275,11 +279,14 @@
                        'verification-done-%s' % release in t) and
                       'verification-needed' in t):
                     cls += ' verificationpartial'
+                    removable = False
                 elif ('verification-done' in t or
                       'verification-done-%s' % release in t):
                     cls += ' verified'
+                    removable = False
                 elif b in broken_bugs:
                     cls += ' broken'
+                    removable = False
                 else:
                     try:
                         bug = lp.bugs[b]
@@ -299,6 +306,7 @@
                                     if (m_date.replace(tzinfo=None) < today
                                             - datetime.timedelta(16)):
                                         cls += ' removal'
+                                        antique = True
                                         continue
                                     hover_text = ''
                                     if 'messages' in cls:
@@ -316,6 +324,7 @@
                                             'UTF-8') + ' - '
                                         hover_text += m_owner.name.encode(
                                             'UTF-8')
+                                        antique = False
                                 except ClientError as error:
                                     # people who don't use lp anymore
                                     if error == 'Gone':
@@ -330,6 +339,9 @@
                       'title="%s" %s>#%d%s</a>' %
                       (b, hover_text.replace('"', ''), cls, b,
                        '(hw)' if 'hw-specific' in t else ''))
+            if antique and removable:
+                proposed_antique.append((releases[release].name, pkg,
+                                        [str(b) for b in rpkg['bugs']]))
             print('&nbsp;</td>')
             print('  <td>%i</td></tr>' % age)
         print('</table>')
@@ -412,6 +424,17 @@
                     '-e %s %s' % (r, pkg[2]['proposed'], pkg[1]))
     print('</pre>')
 
+    print('<p>The following packages have not had their SRU bugs verified in '
+          '105 days and should be removed from -proposed:</p>')
+
+    print('<pre>')
+    for r in releases:
+        for pkg in sorted(proposed_antique):
+            if pkg[0].startswith(r):
+                print('sru-remove -s %s -p %s %s' %
+                      (r, pkg[1], ' '.join(pkg[2])))
+    print('</pre>')
+
     #
     # kernel PPA packages
     #
@@ -639,7 +662,7 @@
 
     kernels = defaultdict(dict)
     for release in releases:
-        #if release != 'karmic': continue # for quick testing
+        # if release != 'karmic': continue # for quick testing
         for published in ppa.getPublishedSources(
                 status='Published', distro_series=releases[release]):
             logging.debug(