Merge ~racb/ubuntu-sponsoring:git-mps into ubuntu-sponsoring:master

[Robie Basak]

Add support for git-ubuntu MPs to appear in the sponsorship reports.

[Simon Quigley (tsimonq2)]

Hi Robie!

First off, thank you for submitting this MR. The Ubuntu Sponsoring Report tends to not get a lot of love these days.

I think this PR looks good, but the fact that it's manually scraping a webpage concerns me. I'd appreciate it if we could get that addressed sooner rather than later.

Lastly, ...why 12.04? Why is that version still in production? That leaves me with many questions... :P

Merging. You may have to poke someone to git pull it on the prod server, I forget exactly how that's handled.

review: Approve

diff --git a/lp_scrape_mps.py b/lp_scrape_mps.py
new file mode 100644
index 0000000..81ff0b3
--- /dev/null
+++ b/lp_scrape_mps.py
@@ -0,0 +1,68 @@
+"""Determine the set of MPs relevant to a particular Launchpad user or team
+
+This works around https://bugs.launchpad.net/launchpad/+bug/1979817 by scraping
+the Launchpad Web UI instead.
+
+Since cranberry (the machine that generates
+http://reqorts.qa.ubuntu.com/reports/sponsoring/index.html) is running 12.04
+and uses Python 2 to generate the report, this file must be kept Python 2.7
+compatible.
+"""
+
+# COPIED FROM lp:~ubuntu-servers/+git/ubuntu-helpers/rbasak/lp_scrape_mps.py
+# Commit: 4237a5a
+
+import itertools
+import re
+
+try:
+    import urllib.request as urllib_request
+except ImportError:
+    # Python 2 compatibility shim
+    import urllib2 as urllib_request
+
+import bs4
+
+CANONICAL_SERVER_MP_URL = (
+    "https://code.launchpad.net/~canonical-server-reporter/+activereviews"
+)
+GIT_UBUNTU_MP_URL = "https://code.launchpad.net/~usd-import-team/usd-importer/+git/usd-importer/+ref/master/+activereviews"
+MATCH_RE = re.compile(r"^/.*/\+merge/[\d]+$")
+MATCH_CAN_DO = re.compile(r"^(Requested reviews)|(Reviews) .* can do$")
+
+
+def _scrape_flat_mps(lp, soup):
+    current_title = None
+    for tag in soup(
+        lambda t: (t.name == "td" and "section-heading" in t.get("class", []))
+        or (t.name == "a" and MATCH_RE.search(t.get("href", "")))
+    ):
+        if tag.name == "td":
+            current_title = tag.string
+        else:
+            yield current_title, lp.load(
+                "https://api.launchpad.net/devel" + tag["href"]
+            )
+
+
+def scrape_mps(lp, web_url):
+    # On Python 2.7, urllib2.urlopen() doesn't return a context manager, so
+    # this must be handled manually.
+    f = urllib_request.urlopen(web_url)
+    try:
+        soup = bs4.BeautifulSoup(f, features="lxml")
+    finally:
+        f.close()
+    return {
+        key: [pair[1] for pair in mp_pairs]
+        for key, mp_pairs in itertools.groupby(
+            _scrape_flat_mps(lp, soup), key=lambda r: r[0]
+        )
+    }
+
+
+def filter_can_do(mps):
+    for k, v in mps.items():
+        if MATCH_CAN_DO.search(k):
+            return v
+    return []
diff --git a/sponsors-page.py b/sponsors-page.py
index 00c7114..33724b1 100755
--- a/sponsors-page.py
+++ b/sponsors-page.py
@@ -13,6 +13,8 @@ import sys
 from lazr.restfulclient.errors import NotFound, ServerError
 import launchpad
 
+import lp_scrape_mps
+
 # { a : b }
 # Look for merge proposals / bug subcriptions for a and output them to b's
 # page. The default (None) is to use a.
@@ -43,6 +45,14 @@ BLACKLISTED_REQUESTER = (
 # A package that is outside of main and package sets
 UNIVERSE_PACKAGE = "mathjax-docs"
 
+GIT_UBUNTU_REPOSITORY_OWNER = (
+    "https://api.launchpad.net/devel/~git-ubuntu-import"
+)
+LP_DISTRIBUTION_SOURCE_PACKAGE_RESOURCE_TYPE = (
+    "https://api.launchpad.net/devel/#distribution_source_package"
+)
+LP_UBUNTU_DISTRIBUTION = "https://api.launchpad.net/devel/ubuntu"
+
 
 def htmlify_name(person, package, devs, distribution):
     if not person:
@@ -382,7 +392,7 @@ def get_bugs(lp, distribution, team, verbose=False):
     active_releases = [a for a in distribution.series if a.active] + \
                       [distribution]
     tasks = set()
-    lp_team = lp.people[SPONSOR_TEAMS[team]]
+    lp_team = lp.people[team]
     for release in active_releases:
         release_name = release.name
         release_tasks = release.searchTasks(bug_subscriber=lp_team,
@@ -513,6 +523,133 @@ def get_branches(lp, distribution, team, verbose=False):
     return tasks
 
 
+def get_git_ubuntu_target_series(ubuntu, target_git_path):
+    if target_git_path.startswith("refs/heads/ubuntu/"):
+        effective_target_git_path = target_git_path
+    elif target_git_path == "refs/heads/debian/sid":
+        # We use "debian/sid" for package merge MPs. LP: 1976112
+        effective_target_git_path = "refs/heads/ubuntu/devel"
+    else:
+        raise ValueError
+    assert effective_target_git_path.startswith("refs/heads/ubuntu/")
+    branch = effective_target_git_path[len("refs/heads/ubuntu/") :]
+    current_series = ubuntu.current_series
+    if branch == "devel":
+        return current_series.name
+    else:
+        try:
+            series, _ = branch.split("-")
+        except ValueError:
+            # target is the "release pocket branch" of a stable series; assume
+            # this is an SRU really intended for -updates via -proposed.
+            return branch
+        return series
+
+
+def get_git_branches(lp, distribution, team, verbose=False):
+    ubuntu = lp.distributions['ubuntu']
+    tasks = set()
+    web_url = (
+        "https://code.launchpad.net/~%s/+activereviews"
+        % team
+    )
+    proposals = [
+        p for p
+        in lp_scrape_mps.filter_can_do(lp_scrape_mps.scrape_mps(
+            lp=lp,
+            web_url=web_url,
+        ))
+        if p.registrant.name not in BLACKLISTED_REQUESTER
+    ]
+    for proposal in proposals:
+        if not proposal.source_git_repository:
+            # Not git (eg. bzr)
+            continue
+        target_repo = proposal.target_git_repository
+        target = target_repo.target
+        if (
+            target.resource_type_link
+            != LP_DISTRIBUTION_SOURCE_PACKAGE_RESOURCE_TYPE
+        ):
+            # Not targetted at a distribution package
+            continue
+        if target.distribution_link != LP_UBUNTU_DISTRIBUTION:
+            # Not targetted at Ubuntu
+            continue
+
+        distribution = target.distribution
+
+        # We need to identify the package and target series. For git-ubuntu
+        # branches, this is straightforward as the branch name schema is known,
+        # so for now we just support these. To support non git-ubuntu git merge
+        # proposals, we would need some consistent way of determining the
+        # target series from the target branch.
+        if target_repo.owner_link != GIT_UBUNTU_REPOSITORY_OWNER:
+            continue
+
+        try:
+            series = get_git_ubuntu_target_series(
+                ubuntu=distribution,
+                target_git_path=proposal.target_git_path,
+            )
+        except ValueError:
+            continue
+        # releases, as supplied by kwargs to SponsoringItem(), seems to mean
+        # the set of stable releases for an SRU, or empty if the development
+        # release.
+        if series != distribution.current_series.name:
+            releases = set([series])
+        else:
+            releases = set()
+
+        s_types = set()
+        status = ""
+        last_comment = ""
+        package = target.name
+        tags = set()
+        try:
+            comments = [a for a in proposal.all_comments]
+        except NotFound:
+            comments = []
+        if comments:
+            status = ["%s comments" % (len(comments))]
+            last_comment = comments[-1].author
+        else:
+            last_comment = proposal.registrant
+        try:
+            votes = [a for a in proposal.votes if a.comment is not None]
+        except NotFound:
+            votes = []
+        if votes:
+            status += [a.comment.vote for a in votes]
+        if proposal.source_git_path.startswith('refs/heads/'):
+            link_desc = proposal.source_git_path[len('refs/heads/'):]
+        else:
+            link_desc = 'MP source branch name unavailable'
+        title = "%s:%s" % (
+            proposal.source_git_repository.display_name,
+            link_desc,
+        )
+        tasks.add(SponsoringItem(
+            distribution=distribution,
+            team=team,
+            link=proposal.web_link,
+            link_desc=link_desc,
+            release=releases,
+            s_types=s_types,
+            package=package,
+            last_comment=last_comment,
+            status=status,
+            importance="",
+            title=title,
+            tags=tags,
+            date_created=proposal.date_created,
+            date_queued=proposal.date_created,
+            verbose=verbose,
+        ))
+    return tasks
+
+
 def get_packageset_dict(sponsoring_items, packages):
     packageset_dict = dict()
     for item in sponsoring_items:
@@ -616,15 +753,21 @@ def main():
         if (perm.person.is_team and
             perm.permission == "Archive Upload Rights")]) \
         | set(EXTRA_TEAMS.keys())
-    for team in SPONSOR_TEAMS:
+    for team, team_lpid in SPONSOR_TEAMS.iteritems():
         sponsoring_items[team] = get_bugs(
-            lp, distribution, team, options.verbose)
-        branches = get_branches(lp, distribution, team, options.verbose)
+            lp, distribution, team_lpid, options.verbose)
+        branches = (
+            get_branches(lp, distribution, team_lpid, options.verbose)
+            | get_git_branches(lp, distribution, team_lpid, options.verbose)
+        )
         sponsoring_items[team] = \
             sponsoring_items.get(team, set()) | branches
     for team in all_uploaders:
         output_team = EXTRA_TEAMS.get(team, SPONSOR_TEAMS["general"]) or team
-        branches = get_branches(lp, distribution, team, options.verbose)
+        branches = (
+            get_branches(lp, distribution, team, options.verbose)
+            | get_git_branches(lp, distribution, team, options.verbose)
+        )
         sponsoring_items[output_team] = \
             sponsoring_items.get(output_team, set()) | branches