Merge lp:~mwhudson/ubuntu-archive-scripts/add-team-report into lp:ubuntu-archive-scripts

[Michael Hudson-Doyle]

Seemed the easiest approach, happy to be told to do it differently :)

This will require python3-attr and python3-jinja2 to be installed on snakefruit before deployment.

[lp:~mwhudson/ubuntu-archive-scripts/add-team-report]

227. By Michael Hudson-Doyle on 2018-09-11

update script and template

[Steve Langasek (vorlon)]

review: Needs Fixing

[lp:~mwhudson/ubuntu-archive-scripts/add-team-report]

228. By Michael Hudson-Doyle on 2018-09-11

review feedback

[Steve Langasek (vorlon)]

review: Approve

=== added file 'generate-team-p-m'
--- generate-team-p-m	1970-01-01 00:00:00 +0000
+++ generate-team-p-m	2018-09-11 23:51:09 +0000
@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2018 Canonical Ltd
+
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# A copy of the GNU General Public License version 2 is in LICENSE.
+
+import argparse
+from collections import defaultdict
+import datetime
+import json
+import os
+import threading
+from urllib.request import urlopen
+
+import attr
+from jinja2 import Environment, FileSystemLoader
+import yaml
+
+env = Environment(
+    loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__)) + '/templates'),
+    autoescape=True,
+    extensions=['jinja2.ext.i18n'],
+)
+env.install_null_translations(True)
+
+_lps = {}
+
+def get_lp(i, anon=True):
+    from launchpadlib.launchpad import Launchpad
+    k = (i, anon)
+    if k not in _lps:
+        print(i, "connecting...")
+        if anon:
+            _lps[k] = Launchpad.login_anonymously('sru-team-report', 'production', version='devel')
+        else:
+            _lps[k] = Launchpad.login_with('sru-team-report', 'production', version='devel')
+    return _lps[k]
+
+
+def get_true_ages_in_proposed(package_names, thread_count):
+    package_names = set(package_names)
+    r = {}
+    def run(i):
+        lp = get_lp(i, True)
+        ubuntu = lp.distributions['ubuntu']
+        primary_archive = ubuntu.archives[0]
+        devel_series = ubuntu.getSeries(name_or_version=ubuntu.development_series_alias)
+        while True:
+            try:
+                spn = package_names.pop()
+            except KeyError:
+                return
+            print(i, "getting true age in proposed for", spn)
+            history = primary_archive.getPublishedSources(
+                source_name=spn, distro_series=devel_series, exact_match=True)
+            last_proposed_spph = None
+            for spph in history:
+                if spph.pocket != "Proposed":
+                    break
+                last_proposed_spph = spph
+            if last_proposed_spph is None:
+                continue
+            age = datetime.datetime.now(tz=last_proposed_spph.date_created.tzinfo) - last_proposed_spph.date_created
+            r[spn] = age.total_seconds() / 3600 / 24
+    threads = []
+    for i in range(thread_count):
+        t = threading.Thread(target=run, args=(i,))
+        threads.append(t)
+        t.start()
+    for t in threads:
+        t.join()
+    return r
+
+
+def get_subscribers_lp(packages, thread_count):
+    from lazr.restfulclient.errors import ClientError
+    packages = set(packages)
+    def run(i, subscribers):
+        lp = get_lp(i, False)
+        ubuntu = lp.distributions['ubuntu']
+        while True:
+            try:
+                spn = packages.pop()
+            except KeyError:
+                return
+            print(i, spn)
+            distribution_source_package = ubuntu.getSourcePackage(name=spn)
+            for subscription in distribution_source_package.getSubscriptions():
+                subscriber = subscription.subscriber
+                try:
+                    if subscriber.is_team:
+                        subscribers[spn].append(subscriber.name)
+                except ClientError:
+                    # This happens for suspended users
+                    pass
+    results = []
+    threads = []
+    for i in range(thread_count):
+        d = defaultdict(list)
+        t = threading.Thread(target=run, args=(i, d))
+        results.append(d)
+        threads.append(t)
+        t.start()
+    for t in threads:
+        t.join()
+    result = defaultdict(list)
+    for d in results:
+        for k, v in d.items():
+            result[k].extend(v)
+    return result
+
+def get_subscribers_json(packages, subscribers_json):
+    if subscribers_json is None:
+        j = urlopen("http://people.canonical.com/~ubuntu-archive/package-team-mapping.json")
+    else:
+        j = open(subscribers_json, 'rb')
+    with j:
+        team_to_packages = json.loads(j.read().decode('utf-8'))
+    package_to_teams = {}
+    for team, packages in team_to_packages.items():
+        for package in packages:
+            package_to_teams.setdefault(package, []).append(team)
+    return package_to_teams
+
+@attr.s
+class ArchRegression:
+    arch = attr.ib(default=None)
+    log_link = attr.ib(default=None)
+    hist_link = attr.ib(default=None)
+
+
+@attr.s
+class Regression:
+    blocking = attr.ib(default=None) # source package name blocked
+    package = attr.ib(default=None) # source package name doing the blocking
+    version = attr.ib(default=None) # version that regressed
+    arches = attr.ib(default=None) # [ArchRegression]
+
+    @property
+    def package_version(self):
+        return self.package + '/' + self.version
+
+@attr.s
+class Problem:
+    kind = attr.ib(default=None) # 'blocked-in-proposed', 'regressing-other'
+    package_in_proposed = attr.ib(default=None) # name of package that's in proposed
+    regressing_package = attr.ib(default=None) # name of package regressing package_in_proposed, None if blocked-in-proposed
+    regressions = attr.ib(default=None) # [Regression]
+    waiting = attr.ib(default=None) # [(source_package_name, arches)]
+    data = attr.ib(default=None) # data for package_in_proposed
+    unsatdepends = attr.ib(default=None) # [string]
+
+    _age = attr.ib(default=None)
+
+    @property
+    def late(self):
+        return self.age > 3
+
+    @property
+    def age(self):
+        if self._age is not None:
+            return self._age
+        else:
+            return self.data["policy_info"]["age"]["current-age"]
+    @age.setter
+    def age(self, val):
+        self._age = val
+
+    @property
+    def key_package(self):
+        if self.regressing_package:
+            return self.regressing_package
+        return self.package_in_proposed
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--ppa', action='store')
+    parser.add_argument('--components', action='store', default="main,restricted")
+    parser.add_argument('--subscribers-from-lp', action='store_true')
+    parser.add_argument('--subscribers-json', action='store')
+    parser.add_argument('--true-ages', action='store_true')
+    parser.add_argument('--excuses-yaml', action='store')
+    parser.add_argument('output')
+    args = parser.parse_args()
+
+    components = args.components.split(',')
+
+    print("fetching yaml")
+    if args.excuses_yaml:
+        yaml_text = open(args.excuses_yaml).read()
+    else:
+        yaml_text = urlopen("https://people.canonical.com/~ubuntu-archive/proposed-migration/update_excuses.yaml").read()
+    print("parsing yaml")
+    # The CSafeLoader is ten times faster than the regular one
+    excuses = yaml.load(yaml_text, Loader=yaml.CSafeLoader)
+
+    print("pre-processing packages")
+    in_proposed_packages = {}
+    for item in excuses["sources"]:
+        source_package_name = item['item-name']
+        # Missing component means main
+        if item.get('component', 'main') not in components:
+            continue
+        prob = Problem(kind='package-in-proposed', data=item, package_in_proposed=source_package_name)
+        in_proposed_packages[source_package_name] = prob
+        prob.regressions = []
+        prob.waiting = []
+        if 'autopkgtest' in item['reason']:
+            for package, results in sorted(item['policy_info']['autopkgtest'].items()):
+                regr_arches = []
+                wait_arches = []
+                for arch, result in sorted(results.items()):
+                    outcome, log, history, wtf1, wtf2 = result
+                    if outcome == "REGRESSION":
+                        regr_arches.append(ArchRegression(arch=arch, log_link=log, hist_link=history))
+                    if outcome == "RUNNING":
+                        wait_arches.append(arch)
+                if regr_arches:
+                    p, v = package.split('/')
+                    regr = Regression(package=p, version=v, blocking=source_package_name)
+                    regr.arches = regr_arches
+                    prob.regressions.append(regr)
+                if wait_arches:
+                    prob.waiting.append((package + ": " + ", ".join(wait_arches)))
+        if 'dependencies' in item and 'unsatisfiable-dependencies' in item['dependencies']:
+                unsatd = defaultdict(list)
+                for arch, packages in item['dependencies']['unsatisfiable-dependencies'].items():
+                    for p in packages:
+                        unsatd[p].append(arch)
+                prob.unsatdepends = ['{}: {}'.format(p, ', '.join(sorted(arches))) for p, arches in sorted(unsatd.items())]
+
+    package_to_problems = defaultdict(list)
+
+    for problem in in_proposed_packages.values():
+        package_to_problems[problem.package_in_proposed].append(problem)
+        for regression in problem.regressions:
+            if regression.blocking not in in_proposed_packages:
+                continue
+            if regression.blocking == regression.package:
+                continue
+            package_to_problems[regression.package].append(Problem(
+                kind='regressing-other', package_in_proposed=regression.blocking,
+                regressing_package=regression.package,
+                regressions=[regression],
+                data=in_proposed_packages[regression.blocking].data))
+
+    if args.true_ages:
+        true_ages = get_true_ages_in_proposed(set(package_to_problems), 10)
+        for package, true_age in true_ages.items():
+            for problem in package_to_problems[package]:
+                problem.age = true_age
+
+    print("getting subscribers")
+    if args.subscribers_from_lp:
+        subscribers = get_subscribers_lp(set(package_to_problems), 10)
+        for p in set(package_to_problems):
+            if p not in subscribers:
+                subscribers[p] = ['unsubscribed']
+    else:
+        subscribers = get_subscribers_json(set(package_to_problems), args.subscribers_json)
+        for p in set(package_to_problems):
+            if p not in subscribers:
+                subscribers[p] = ['unknown']
+
+    all_teams = set()
+    team_to_problems = defaultdict(list)
+    for package, teams in subscribers.items():
+        all_teams |= set(teams)
+        for team in teams:
+            team_to_problems[team].extend(package_to_problems[package])
+
+    for packages in team_to_problems.values():
+        packages.sort(key=lambda prob: (-prob.age, prob.key_package))
+
+    team_to_attn_count = {}
+    for team, problems in team_to_problems.items():
+        team_to_attn_count[team] = len([problem for problem in problems if problem.late])
+
+    print("rendering")
+    t = env.get_template('team-report.html')
+    with open(args.output, 'w', encoding='utf-8') as fp:
+        fp.write(t.render(
+            all_teams=all_teams,
+            team_to_problems=team_to_problems,
+            team_to_attn_count=team_to_attn_count,
+            now=excuses["generated-date"].strftime("%Y.%m.%d %H:%M:%S")))
+
+if __name__ == '__main__':
+    main()

=== modified file 'run-proposed-migration'
--- run-proposed-migration	2018-05-01 17:36:23 +0000
+++ run-proposed-migration	2018-09-11 23:51:09 +0000
@@ -47,3 +47,8 @@
 mkdir -p "${logfile%/*}"
 
 code/b1/britney $actions >"$logfile" 2>&1
+if [ "$SERIES" = "$DEFAULT_SERIES" ]; then
+    generate-team-p-m --excuses-yaml $SERIES/update_excuses.yaml \
+                      --subscribers-json ~/public_html/package-team-mapping.json \
+                      $SERIES/update_excuses_by_team.html
+fi

=== added directory 'templates'
=== added file 'templates/team-report.html'
--- templates/team-report.html	1970-01-01 00:00:00 +0000
+++ templates/team-report.html	2018-09-11 23:51:09 +0000
@@ -0,0 +1,107 @@
+<!DOCTYPE HTML>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <title>devel-proposed by team</title>
+    <style>
+      body { font-family: ubuntu }
+      .not-late { color: grey; }
+      .not-late a { color: #77f; }
+    </style>
+  </head>
+  <body lang="en">
+    <h1>devel-proposed by team</h1>
+    <p>
+      Generated: {{ now }}
+    <ul>
+      {% for team in all_teams|sort %}
+      <li><a href="#{{ team }}">{{ team }}</a> ({{ ngettext("%(num)d package", "%(num)d packages", team_to_attn_count[team]) }} needing attention)</li>
+      {% endfor %}
+    </ul>
+    {% for team in all_teams|sort %}
+    <h1 id="{{ team }}">{{ team }}</h1>
+    <p>
+      {{ ngettext("%(num)d package", "%(num)d packages", team_to_attn_count[team]) }} needing attention, {{ ngettext("%(num)d package", "%(num)d packages", team_to_problems[team]|count - team_to_attn_count[team]) }} not yet considered late
+    </p>
+    <ul>
+      {% for prob in team_to_problems[team] %}
+      {% set d = prob.data %}
+      {% set p = prob.package_in_proposed %}
+      <li {% if not prob.late %}class="not-late"{% endif %}>
+        {% if prob.kind == "regressing-other" %}
+        <b>{{ prob.regressing_package }}</b> blocking
+        <a href="http://people.canonical.com/~ubuntu-archive/proposed-migration/update_excuses.html#{{ p }}">{{ p }}</a>
+        {% else %}
+        <a href="http://people.canonical.com/~ubuntu-archive/proposed-migration/update_excuses.html#{{ p }}"><b>{{ p }}</b></a>
+        {% endif %}
+        {% set urlbase = "https://launchpad.net/ubuntu/+source/" + p + "/" %}
+        ({% if d['old-version'] != "-" %}<a href="{{ urlbase }}{{ d["old-version"] }}">{{ d["old-version"] }}{% else %}-{% endif %}</a>
+        to
+        <a href="{{ urlbase }}{{ d["new-version"] }}">{{ d["new-version"] }}</a>)
+        {% if prob.kind == "regressing-other" %}
+        for
+        {% else %}
+        in proposed for
+        {% endif %}
+        {{ ngettext("%(num)d day", "%(num)d days", prob.age|int) }}
+        <ul>
+          {% if prob.waiting %}
+          <li>
+            Waiting
+            <ul>
+              {% for p in prob.waiting %}
+              <li>{{ p }}</li>
+              {% endfor %}
+            </ul>
+          </li>
+          {% endif %}
+          {% if d.get("missing-builds") %}
+          <li>Missing builds, see excuses</li>
+          {% endif %}
+          {% if 'no-binaries' in d["reason"] %}
+          <li>No binaries</li>
+          {% endif %}
+          {% if prob.regressions %}
+          <li>
+            Regressions
+            <ul>
+              {% for regr in prob.regressions %}
+              <li>
+                {{ regr.package_version }}:
+                {% for arch_regr in regr.arches %}
+                {{ arch_regr.arch }} (<a href="{{ arch_regr.log_link }}">log</a>, <a href="{{ arch_regr.hist_link }}">history</a>)
+                {%- if not loop.last %}, {% endif %}
+                {% endfor %}
+              </li>
+              {% endfor %}
+            </ul>
+          </li>
+          {% endif %}
+          {% if prob.kind != "regressing-other" %}
+          {% if d.get('dependencies', {}).get('blocked-by', []) %}
+          <li>Depends on {{ d['dependencies']['blocked-by']|join(", ") }}</li>
+          {% endif %}
+          {% if prob.unsatdepends %}
+          <li>
+            Unsatisfiable depends:
+            <ul>
+              {% for unsatd in prob.unsatdepends %}
+              <li>{{ unsatd }}</li>
+              {% endfor %}
+            </ul>
+          </li>
+          {% endif %}
+          {% if d["policy_info"]["block-bugs"] %}
+          <li>Blocked by bug: {{ d["policy_info"]["block-bugs"]|map("int")|sort|map("string")|join(", ") }}</li>
+          {% endif %}
+          {% if d["is-candidate"] %}
+          <li><b>candidate</b></li>
+          {% endif %}
+          {% endif %}
+        </ul>
+      </li>
+      {% endfor %}
+    </ul>
+    {% endfor %}
+  </body>
+</html>