Merge lp:~mwhudson/ubuntu-archive-scripts/add-team-report into lp:ubuntu-archive-scripts

Proposed by Michael Hudson-Doyle
Status: Merged
Merged at revision: 226
Proposed branch: lp:~mwhudson/ubuntu-archive-scripts/add-team-report
Merge into: lp:ubuntu-archive-scripts
Diff against target: 431 lines (+413/-0)
3 files modified
generate-team-p-m (+301/-0)
run-proposed-migration (+5/-0)
templates/team-report.html (+107/-0)
To merge this branch: bzr merge lp:~mwhudson/ubuntu-archive-scripts/add-team-report
Reviewer Review Type Date Requested Status
Steve Langasek Approve
Review via email: mp+354278@code.launchpad.net

Description of the change

Seemed the easiest approach, happy to be told to do it differently :)

This will require python3-attr and python3-jinja2 to be installed on snakefruit before deployment.

To post a comment you must log in.
227. By Michael Hudson-Doyle

update script and template

Revision history for this message
Steve Langasek (vorlon) :
review: Needs Fixing
228. By Michael Hudson-Doyle

review feedback

Revision history for this message
Steve Langasek (vorlon) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added file 'generate-team-p-m'
2--- generate-team-p-m 1970-01-01 00:00:00 +0000
3+++ generate-team-p-m 2018-09-11 23:51:09 +0000
4@@ -0,0 +1,301 @@
5+#!/usr/bin/env python3
6+# -*- coding: utf-8 -*-
7+
8+# Copyright (C) 2018 Canonical Ltd
9+
10+# This program is free software; you can redistribute it and/or
11+# modify it under the terms of the GNU General Public License
12+# as published by the Free Software Foundation; either version 2
13+# of the License, or (at your option) any later version.
14+#
15+# This program is distributed in the hope that it will be useful,
16+# but WITHOUT ANY WARRANTY; without even the implied warranty of
17+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18+# GNU General Public License for more details.
19+#
20+# A copy of the GNU General Public License version 2 is in LICENSE.
21+
22+import argparse
23+from collections import defaultdict
24+import datetime
25+import json
26+import os
27+import threading
28+from urllib.request import urlopen
29+
30+import attr
31+from jinja2 import Environment, FileSystemLoader
32+import yaml
33+
34+env = Environment(
35+ loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__)) + '/templates'),
36+ autoescape=True,
37+ extensions=['jinja2.ext.i18n'],
38+)
39+env.install_null_translations(True)
40+
41+_lps = {}
42+
43+def get_lp(i, anon=True):
44+ from launchpadlib.launchpad import Launchpad
45+ k = (i, anon)
46+ if k not in _lps:
47+ print(i, "connecting...")
48+ if anon:
49+ _lps[k] = Launchpad.login_anonymously('sru-team-report', 'production', version='devel')
50+ else:
51+ _lps[k] = Launchpad.login_with('sru-team-report', 'production', version='devel')
52+ return _lps[k]
53+
54+
55+def get_true_ages_in_proposed(package_names, thread_count):
56+ package_names = set(package_names)
57+ r = {}
58+ def run(i):
59+ lp = get_lp(i, True)
60+ ubuntu = lp.distributions['ubuntu']
61+ primary_archive = ubuntu.archives[0]
62+ devel_series = ubuntu.getSeries(name_or_version=ubuntu.development_series_alias)
63+ while True:
64+ try:
65+ spn = package_names.pop()
66+ except KeyError:
67+ return
68+ print(i, "getting true age in proposed for", spn)
69+ history = primary_archive.getPublishedSources(
70+ source_name=spn, distro_series=devel_series, exact_match=True)
71+ last_proposed_spph = None
72+ for spph in history:
73+ if spph.pocket != "Proposed":
74+ break
75+ last_proposed_spph = spph
76+ if last_proposed_spph is None:
77+ continue
78+ age = datetime.datetime.now(tz=last_proposed_spph.date_created.tzinfo) - last_proposed_spph.date_created
79+ r[spn] = age.total_seconds() / 3600 / 24
80+ threads = []
81+ for i in range(thread_count):
82+ t = threading.Thread(target=run, args=(i,))
83+ threads.append(t)
84+ t.start()
85+ for t in threads:
86+ t.join()
87+ return r
88+
89+
90+def get_subscribers_lp(packages, thread_count):
91+ from lazr.restfulclient.errors import ClientError
92+ packages = set(packages)
93+ def run(i, subscribers):
94+ lp = get_lp(i, False)
95+ ubuntu = lp.distributions['ubuntu']
96+ while True:
97+ try:
98+ spn = packages.pop()
99+ except KeyError:
100+ return
101+ print(i, spn)
102+ distribution_source_package = ubuntu.getSourcePackage(name=spn)
103+ for subscription in distribution_source_package.getSubscriptions():
104+ subscriber = subscription.subscriber
105+ try:
106+ if subscriber.is_team:
107+ subscribers[spn].append(subscriber.name)
108+ except ClientError:
109+ # This happens for suspended users
110+ pass
111+ results = []
112+ threads = []
113+ for i in range(thread_count):
114+ d = defaultdict(list)
115+ t = threading.Thread(target=run, args=(i, d))
116+ results.append(d)
117+ threads.append(t)
118+ t.start()
119+ for t in threads:
120+ t.join()
121+ result = defaultdict(list)
122+ for d in results:
123+ for k, v in d.items():
124+ result[k].extend(v)
125+ return result
126+
127+def get_subscribers_json(packages, subscribers_json):
128+ if subscribers_json is None:
129+ j = urlopen("http://people.canonical.com/~ubuntu-archive/package-team-mapping.json")
130+ else:
131+ j = open(subscribers_json, 'rb')
132+ with j:
133+ team_to_packages = json.loads(j.read().decode('utf-8'))
134+ package_to_teams = {}
135+ for team, packages in team_to_packages.items():
136+ for package in packages:
137+ package_to_teams.setdefault(package, []).append(team)
138+ return package_to_teams
139+
140+@attr.s
141+class ArchRegression:
142+ arch = attr.ib(default=None)
143+ log_link = attr.ib(default=None)
144+ hist_link = attr.ib(default=None)
145+
146+
147+@attr.s
148+class Regression:
149+ blocking = attr.ib(default=None) # source package name blocked
150+ package = attr.ib(default=None) # source package name doing the blocking
151+ version = attr.ib(default=None) # version that regressed
152+ arches = attr.ib(default=None) # [ArchRegression]
153+
154+ @property
155+ def package_version(self):
156+ return self.package + '/' + self.version
157+
158+@attr.s
159+class Problem:
160+ kind = attr.ib(default=None) # 'blocked-in-proposed', 'regressing-other'
161+ package_in_proposed = attr.ib(default=None) # name of package that's in proposed
162+ regressing_package = attr.ib(default=None) # name of package regressing package_in_proposed, None if blocked-in-proposed
163+ regressions = attr.ib(default=None) # [Regression]
164+ waiting = attr.ib(default=None) # [(source_package_name, arches)]
165+ data = attr.ib(default=None) # data for package_in_proposed
166+ unsatdepends = attr.ib(default=None) # [string]
167+
168+ _age = attr.ib(default=None)
169+
170+ @property
171+ def late(self):
172+ return self.age > 3
173+
174+ @property
175+ def age(self):
176+ if self._age is not None:
177+ return self._age
178+ else:
179+ return self.data["policy_info"]["age"]["current-age"]
180+ @age.setter
181+ def age(self, val):
182+ self._age = val
183+
184+ @property
185+ def key_package(self):
186+ if self.regressing_package:
187+ return self.regressing_package
188+ return self.package_in_proposed
189+
190+
191+def main():
192+ parser = argparse.ArgumentParser()
193+ parser.add_argument('--ppa', action='store')
194+ parser.add_argument('--components', action='store', default="main,restricted")
195+ parser.add_argument('--subscribers-from-lp', action='store_true')
196+ parser.add_argument('--subscribers-json', action='store')
197+ parser.add_argument('--true-ages', action='store_true')
198+ parser.add_argument('--excuses-yaml', action='store')
199+ parser.add_argument('output')
200+ args = parser.parse_args()
201+
202+ components = args.components.split(',')
203+
204+ print("fetching yaml")
205+ if args.excuses_yaml:
206+ yaml_text = open(args.excuses_yaml).read()
207+ else:
208+ yaml_text = urlopen("https://people.canonical.com/~ubuntu-archive/proposed-migration/update_excuses.yaml").read()
209+ print("parsing yaml")
210+ # The CSafeLoader is ten times faster than the regular one
211+ excuses = yaml.load(yaml_text, Loader=yaml.CSafeLoader)
212+
213+ print("pre-processing packages")
214+ in_proposed_packages = {}
215+ for item in excuses["sources"]:
216+ source_package_name = item['item-name']
217+ # Missing component means main
218+ if item.get('component', 'main') not in components:
219+ continue
220+ prob = Problem(kind='package-in-proposed', data=item, package_in_proposed=source_package_name)
221+ in_proposed_packages[source_package_name] = prob
222+ prob.regressions = []
223+ prob.waiting = []
224+ if 'autopkgtest' in item['reason']:
225+ for package, results in sorted(item['policy_info']['autopkgtest'].items()):
226+ regr_arches = []
227+ wait_arches = []
228+ for arch, result in sorted(results.items()):
229+ outcome, log, history, wtf1, wtf2 = result
230+ if outcome == "REGRESSION":
231+ regr_arches.append(ArchRegression(arch=arch, log_link=log, hist_link=history))
232+ if outcome == "RUNNING":
233+ wait_arches.append(arch)
234+ if regr_arches:
235+ p, v = package.split('/')
236+ regr = Regression(package=p, version=v, blocking=source_package_name)
237+ regr.arches = regr_arches
238+ prob.regressions.append(regr)
239+ if wait_arches:
240+ prob.waiting.append((package + ": " + ", ".join(wait_arches)))
241+ if 'dependencies' in item and 'unsatisfiable-dependencies' in item['dependencies']:
242+ unsatd = defaultdict(list)
243+ for arch, packages in item['dependencies']['unsatisfiable-dependencies'].items():
244+ for p in packages:
245+ unsatd[p].append(arch)
246+ prob.unsatdepends = ['{}: {}'.format(p, ', '.join(sorted(arches))) for p, arches in sorted(unsatd.items())]
247+
248+ package_to_problems = defaultdict(list)
249+
250+ for problem in in_proposed_packages.values():
251+ package_to_problems[problem.package_in_proposed].append(problem)
252+ for regression in problem.regressions:
253+ if regression.blocking not in in_proposed_packages:
254+ continue
255+ if regression.blocking == regression.package:
256+ continue
257+ package_to_problems[regression.package].append(Problem(
258+ kind='regressing-other', package_in_proposed=regression.blocking,
259+ regressing_package=regression.package,
260+ regressions=[regression],
261+ data=in_proposed_packages[regression.blocking].data))
262+
263+ if args.true_ages:
264+ true_ages = get_true_ages_in_proposed(set(package_to_problems), 10)
265+ for package, true_age in true_ages.items():
266+ for problem in package_to_problems[package]:
267+ problem.age = true_age
268+
269+ print("getting subscribers")
270+ if args.subscribers_from_lp:
271+ subscribers = get_subscribers_lp(set(package_to_problems), 10)
272+ for p in set(package_to_problems):
273+ if p not in subscribers:
274+ subscribers[p] = ['unsubscribed']
275+ else:
276+ subscribers = get_subscribers_json(set(package_to_problems), args.subscribers_json)
277+ for p in set(package_to_problems):
278+ if p not in subscribers:
279+ subscribers[p] = ['unknown']
280+
281+ all_teams = set()
282+ team_to_problems = defaultdict(list)
283+ for package, teams in subscribers.items():
284+ all_teams |= set(teams)
285+ for team in teams:
286+ team_to_problems[team].extend(package_to_problems[package])
287+
288+ for packages in team_to_problems.values():
289+ packages.sort(key=lambda prob: (-prob.age, prob.key_package))
290+
291+ team_to_attn_count = {}
292+ for team, problems in team_to_problems.items():
293+ team_to_attn_count[team] = len([problem for problem in problems if problem.late])
294+
295+ print("rendering")
296+ t = env.get_template('team-report.html')
297+ with open(args.output, 'w', encoding='utf-8') as fp:
298+ fp.write(t.render(
299+ all_teams=all_teams,
300+ team_to_problems=team_to_problems,
301+ team_to_attn_count=team_to_attn_count,
302+ now=excuses["generated-date"].strftime("%Y.%m.%d %H:%M:%S")))
303+
304+if __name__ == '__main__':
305+ main()
306
307=== modified file 'run-proposed-migration'
308--- run-proposed-migration 2018-05-01 17:36:23 +0000
309+++ run-proposed-migration 2018-09-11 23:51:09 +0000
310@@ -47,3 +47,8 @@
311 mkdir -p "${logfile%/*}"
312
313 code/b1/britney $actions >"$logfile" 2>&1
314+if [ "$SERIES" = "$DEFAULT_SERIES" ]; then
315+ generate-team-p-m --excuses-yaml $SERIES/update_excuses.yaml \
316+ --subscribers-json ~/public_html/package-team-mapping.json \
317+ $SERIES/update_excuses_by_team.html
318+fi
319
320=== added directory 'templates'
321=== added file 'templates/team-report.html'
322--- templates/team-report.html 1970-01-01 00:00:00 +0000
323+++ templates/team-report.html 2018-09-11 23:51:09 +0000
324@@ -0,0 +1,107 @@
325+<!DOCTYPE HTML>
326+<html>
327+ <head>
328+ <meta charset="utf-8">
329+ <title>devel-proposed by team</title>
330+ <style>
331+ body { font-family: ubuntu }
332+ .not-late { color: grey; }
333+ .not-late a { color: #77f; }
334+ </style>
335+ </head>
336+ <body lang="en">
337+ <h1>devel-proposed by team</h1>
338+ <p>
339+ Generated: {{ now }}
340+ <ul>
341+ {% for team in all_teams|sort %}
342+ <li><a href="#{{ team }}">{{ team }}</a> ({{ ngettext("%(num)d package", "%(num)d packages", team_to_attn_count[team]) }} needing attention)</li>
343+ {% endfor %}
344+ </ul>
345+ {% for team in all_teams|sort %}
346+ <h1 id="{{ team }}">{{ team }}</h1>
347+ <p>
348+ {{ ngettext("%(num)d package", "%(num)d packages", team_to_attn_count[team]) }} needing attention, {{ ngettext("%(num)d package", "%(num)d packages", team_to_problems[team]|count - team_to_attn_count[team]) }} not yet considered late
349+ </p>
350+ <ul>
351+ {% for prob in team_to_problems[team] %}
352+ {% set d = prob.data %}
353+ {% set p = prob.package_in_proposed %}
354+ <li {% if not prob.late %}class="not-late"{% endif %}>
355+ {% if prob.kind == "regressing-other" %}
356+ <b>{{ prob.regressing_package }}</b> blocking
357+ <a href="http://people.canonical.com/~ubuntu-archive/proposed-migration/update_excuses.html#{{ p }}">{{ p }}</a>
358+ {% else %}
359+ <a href="http://people.canonical.com/~ubuntu-archive/proposed-migration/update_excuses.html#{{ p }}"><b>{{ p }}</b></a>
360+ {% endif %}
361+ {% set urlbase = "https://launchpad.net/ubuntu/+source/" + p + "/" %}
362+ ({% if d['old-version'] != "-" %}<a href="{{ urlbase }}{{ d["old-version"] }}">{{ d["old-version"] }}{% else %}-{% endif %}</a>
363+ to
364+ <a href="{{ urlbase }}{{ d["new-version"] }}">{{ d["new-version"] }}</a>)
365+ {% if prob.kind == "regressing-other" %}
366+ for
367+ {% else %}
368+ in proposed for
369+ {% endif %}
370+ {{ ngettext("%(num)d day", "%(num)d days", prob.age|int) }}
371+ <ul>
372+ {% if prob.waiting %}
373+ <li>
374+ Waiting
375+ <ul>
376+ {% for p in prob.waiting %}
377+ <li>{{ p }}</li>
378+ {% endfor %}
379+ </ul>
380+ </li>
381+ {% endif %}
382+ {% if d.get("missing-builds") %}
383+ <li>Missing builds, see excuses</li>
384+ {% endif %}
385+ {% if 'no-binaries' in d["reason"] %}
386+ <li>No binaries</li>
387+ {% endif %}
388+ {% if prob.regressions %}
389+ <li>
390+ Regressions
391+ <ul>
392+ {% for regr in prob.regressions %}
393+ <li>
394+ {{ regr.package_version }}:
395+ {% for arch_regr in regr.arches %}
396+ {{ arch_regr.arch }} (<a href="{{ arch_regr.log_link }}">log</a>, <a href="{{ arch_regr.hist_link }}">history</a>)
397+ {%- if not loop.last %}, {% endif %}
398+ {% endfor %}
399+ </li>
400+ {% endfor %}
401+ </ul>
402+ </li>
403+ {% endif %}
404+ {% if prob.kind != "regressing-other" %}
405+ {% if d.get('dependencies', {}).get('blocked-by', []) %}
406+ <li>Depends on {{ d['dependencies']['blocked-by']|join(", ") }}</li>
407+ {% endif %}
408+ {% if prob.unsatdepends %}
409+ <li>
410+ Unsatisfiable depends:
411+ <ul>
412+ {% for unsatd in prob.unsatdepends %}
413+ <li>{{ unsatd }}</li>
414+ {% endfor %}
415+ </ul>
416+ </li>
417+ {% endif %}
418+ {% if d["policy_info"]["block-bugs"] %}
419+ <li>Blocked by bug: {{ d["policy_info"]["block-bugs"]|map("int")|sort|map("string")|join(", ") }}</li>
420+ {% endif %}
421+ {% if d["is-candidate"] %}
422+ <li><b>candidate</b></li>
423+ {% endif %}
424+ {% endif %}
425+ </ul>
426+ </li>
427+ {% endfor %}
428+ </ul>
429+ {% endfor %}
430+ </body>
431+</html>

Subscribers

People subscribed via source and target branches