Merge lp:~jibel/canonical-qa-tracking/sru-tools into lp:canonical-qa-tracking

[Jean-Baptiste Lallement]

Moving the thankyou script from ubuntu-qa-tools to canonical-qa-tracking following comment of bdmurray https://code.edge.launchpad.net/~jibel/ubuntu-qa-tools/thankyou.xlpb-modifiers/+merge/34522

[Brian Murray (brian-murray)]

review: Approve

=== added directory 'sru-tools'
=== added file 'sru-tools/thankyou-testers'
--- sru-tools/thankyou-testers	1970-01-01 00:00:00 +0000
+++ sru-tools/thankyou-testers	2010-09-24 14:01:01 +0000
@@ -0,0 +1,239 @@
+#!/usr/bin/python
+#
+# This script will parse a bugs mailing list file for e-mails indicating
+# that a bug has been tagged with a verification-* tag
+# We then export the list of users having added a comment after between the
+# moment the report has been tagged verification-needed and the moment the
+# report has been tagged verification-done
+# Canonical employees are filtered out
+#
+# The output are 2 "thank you" messages, one for IRC and I for the Wiki
+# The messages are saved to /tmp/sru_thankyou.XXXXXXXX.msg for further reading
+# use mutt -f /tmp/sru_thankyou.XXXXXXXX.msg
+#
+# Example:
+# sru-thankyou 2010-08-25 bugmail/2010-08 bugmail/2010-09
+#
+#
+# Copyright 2008-2010 Canonical, Ltd
+# Author: Jean-Baptiste Lallement <[e-mail]>
+# Credits to Brian Murray for the parsing of the bugmail archive.
+# Licensed under the GNU General Public License, version 3.
+
+# TODO:
+#   - Sort people by number of tasks verified
+#
+# We use 2 structures as to store bug and person information:
+# srubugs[bug_id]
+#   0. displayname
+#   1. email addr
+#   2. (display_name, lp_id)     From X-Launchpad-Bug-Modifier
+#   3. action_date
+#
+# people[lpid]
+#  0. displayname
+#  1. bugs[]        list of bugs commented by this person
+#  2. ircnickname
+#  3. wikiname
+#  4. canonical     True if it's a canonical employee. See isCanonicalEmployee()
+#
+from mailbox import PortableUnixMailbox
+from email import message_from_file
+from email.errors import MessageParseError
+from email.utils import parseaddr
+from email.utils import parsedate
+from sys import argv, stderr, stdout, exit
+from operator import itemgetter
+import logging
+import re
+
+import time
+from launchpadlib.launchpad import Launchpad
+
+LOGGINGLEVEL = logging.DEBUG
+xlpm_pattern = re.compile('(?P<name>.+) \((?P<id>[^\)]+)\)')
+
+lp = None
+# adjust as appropriate
+report_folder = ""
+
+MESSAGE_NOT_PARSEABLE = object()
+
+def message_factory(fp):
+    try:
+        return message_from_file(fp)
+    except MessageParseError:
+        # Don't return None since that will stop the mailbox iterator.
+        return MESSAGE_NOT_PARSEABLE
+
+
+def show_progress(iterator, interval=100):
+    """Show signs of progress."""
+    for count, item in enumerate(iterator):
+        if count % interval == 0:
+            stderr.write('.')
+        yield item
+
+def scan_bugs(startdate, messagelog, messages):
+    """ Scan interesting reports
+
+    startdate: Date from which we want to export messages
+    messagelog: file descriptor where to store the interesting messages for
+        further reading
+    """
+    global xlpm_pattern
+
+    srudone = []
+    for count, message in enumerate(messages):
+        # Skip broken messages.
+        if message is MESSAGE_NOT_PARSEABLE:
+            continue
+
+        # Check it's from a Launchpad bug.
+        reply_to = message['reply-to']
+        if reply_to is None:
+            continue
+        reply_name, reply_address = parseaddr(reply_to)
+        reply_local_part, reply_domain = reply_address.split('@')
+        if not (reply_domain == 'bugs.launchpad.net' and
+                reply_local_part.isdigit()):
+            continue
+        tags = message['X-Launchpad-Bug-Tags']
+
+        bug_id = int(reply_local_part)
+        # Comments after verification is done are often metooing and not very useful
+        if tags and 'verification-' in tags and not bug_id in srudone:
+            sent_date = parsedate(message['date'])
+            if sent_date < startdate:
+                continue
+            sender = parseaddr(message['from'])
+
+            # Remove the robots
+            if sender[0] in ('Launchpad Bug Tracker',
+                             'Bug Watch Updater',
+                             'Ubuntu QA Website'):
+                continue
+            xlpm_header = message['X-Launchpad-Bug-Modifier']
+            if not xlpm_header:
+                continue
+            xlpm_m = xlpm_pattern.search(xlpm_header)
+            if xlpm_m:
+                modifier=xlpm_m.groupdict() # {display_name, lp_id}
+            action_date = time.mktime(sent_date)
+            try:
+                srubugs[bug_id].append((sender[0], sender[1], modifier, action_date))
+            except KeyError:
+                srubugs[bug_id] = [(sender[0], sender[1], modifier, action_date)]
+
+
+            messagelog.write(str(message))
+            if tags and 'verification-done' in tags:
+                srudone.append(bug_id)
+
+def ircnickname(person, server="irc.freenode.net"):
+    for irc in person.irc_nicknames:
+        if irc.network == server:
+            return irc.nickname
+    return None
+
+def wikiname(person, server='wiki.ubuntu.com'):
+    for wiki in person.wiki_names:
+        if server in wiki.wiki:
+            return wiki.wikiname
+    return None
+
+def isCanonicalEmployee(person):
+    """ Return true if we think the person is a Canonical employee
+
+    We assume that any member of a canonical team is a canonical employee
+    Is there a way to find if a team is a subteam of canonical ???
+    """
+    canonical_teams = ('canonical', 'landscape')
+    for tm in person.memberships_details:
+        for ct in canonical_teams:
+            if ct in tm.team.name:
+                return True
+    return False
+
+def printReport(people, type ):
+    if not people:
+        print "No people to thank. Sorry!"
+        return
+
+    print "== Thank you output for %s ==" % ( 'IRC' if 'irc' in type.lower() else "WIKI")
+    # Filter Canonical employees
+    noncanonical = filter(lambda x: not people[x]['canonical'] , people)
+
+    if 'irc' in type.lower():
+        thankslist = map(lambda x: people[x]['displayname'] +
+                                ' (' + str(people[x]['ircnickname']) + ')'
+                                if people[x]['ircnickname']
+                                else people[x]['displayname'], sorted(noncanonical))
+    else:
+        thankslist = map(lambda x: '[[' + str(people[x]['wikiname']) +
+                                '|' + people[x]['displayname'] + ']]'
+                                if people[x]['wikiname']
+                                else people[x]['displayname'], sorted(noncanonical))
+
+    print "Thanks to %s and %s for testing packages in -proposed.\n\
+\n\
+As always, you can see the current set of packages needing testing in the \
+-proposed queue at http://people.canonical.com/~ubuntu-archive/pending-sru.html . \
+Your assistance in testing is always appreciated! " % \
+    (", ".join(thankslist[:-1]), thankslist[-1])
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=LOGGINGLEVEL)
+
+    # Check if the amount of arguments is correct
+    if len(argv) < 3 or argv[1] in ('help', '-h', '--help'):
+        print 'Usage: %s <from YYYY-MM-DD> <bug_mailinglist_file> [<bug_mailinglist_file> ...]' % argv[0]
+        exit(1)
+
+    srubugs = {}
+    people = {}
+
+    startdate = time.strptime(argv[1], "%Y-%m-%d")
+    messagelogname = "/tmp/sru_thankyou.%s.msg" % (str(int(time.time())))
+    messagelog = open(messagelogname, "wb")
+
+    for mailbox_file in argv[2:]:
+        mailbox = PortableUnixMailbox(
+            open(mailbox_file, 'rb'), message_factory)
+        scan_bugs(startdate, messagelog, show_progress(mailbox))
+
+    print
+    lp = Launchpad.login_anonymously("lookup-irc-nick", "edge")
+    for k, b in sorted(srubugs.items(), key=itemgetter(1), reverse=False):
+        for v in b:
+            logging.debug('Searching [%s]: %s' % (v[2]['id'],v[2]['name']))
+            personId = v[2]['id']
+            try:
+                people[personId]['bugs'].append(k)
+                logging.debug("- Already exists: %s" % (personId))
+                logging.debug("- Updating with bug: %s" % (k))
+            except KeyError:
+                try:
+                    person = lp.people[personId]
+                except KeyError:
+                    continue
+                if person:
+                    logging.debug("= Found: [%s]	%s" % (v[2]['id'],person.name))
+                    people[person.name] =  {
+                        'displayname':person.display_name,
+                        'bugs':[k],
+                        'ircnickname':ircnickname(person),
+                        'wikiname':wikiname(person),
+                        'canonical':isCanonicalEmployee(person)
+                    }
+                else:
+                    logging.debug("No person found with id [%s] skipping" % personId)
+                    continue
+
+    print
+    printReport(people,'irc')
+    print
+    printReport(people,'wiki')
+    print
+    print "Messages have been saved to ", messagelogname