Merge lp:~rbalint/ubuntu-archive-scripts/bzr-by-team-report into lp:ubuntu-archive-scripts

[Balint Reczey]

[lp:~rbalint/ubuntu-archive-scripts/bzr-by-team-report]

236. By Balint Reczey on 2018-11-23

add link to UbuntuDevelopment/MigratingFromBzrToGit

[Balint Reczey (rbalint)]

I'd also like to add the new report to the reports generated from archive-reports but I'm wondering if there is an easy way of testing that or someone with a setup for testing could add it for me.

[Dimitri John Ledkov (xnox)]

PING PING PING PING

[Łukasz Zemczak (sil2100)]

Ok, I guess it should be fine to have such a 'short lived' report for the bazaar migration. Once the bzr migration is complete, it might be good to get rid of this. Code looks okay - at least it follows the same style and design as the generate-team-p-m script. I'll fix some whitespaces and get this merged.

review: Approve

[Łukasz Zemczak (sil2100)]

Oh, I missed a small typo in the template, but now fixed.

=== added file 'generate-team-bzr'
--- generate-team-bzr	1970-01-01 00:00:00 +0000
+++ generate-team-bzr	2018-11-23 17:03:38 +0000
@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2018 Canonical Ltd
+
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# A copy of the GNU General Public License version 2 is in LICENSE.
+
+import apt_pkg
+import argparse
+import atexit
+from collections import defaultdict
+from datetime import datetime
+import gzip
+import json
+import os
+import re
+import shutil
+import tempfile
+from textwrap import dedent
+from urllib.request import urlopen
+
+import attr
+from jinja2 import Environment, FileSystemLoader
+import yaml
+
+env = Environment(
+    loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__)) + '/templates'),
+    autoescape=True,
+    extensions=['jinja2.ext.i18n'],
+)
+env.install_null_translations(True)
+
+tempdir = None
+
+# copied from component-mismatches
+def ensure_tempdir():
+    global tempdir
+    if not tempdir:
+        tempdir = tempfile.mkdtemp(prefix='component-mismatches')
+        atexit.register(shutil.rmtree, tempdir)
+
+
+# copied from component-mismatches
+def decompress_open(tagfile):
+    ensure_tempdir()
+    decompressed = tempfile.mktemp(dir=tempdir)
+    fin = gzip.GzipFile(filename=tagfile)
+    with open(decompressed, 'wb') as fout:
+        fout.write(fin.read())
+    return open(decompressed, 'r')
+
+
+# copied from generate-team-p-m
+def get_subscribers_json(packages, subscribers_json):
+    if subscribers_json is None:
+        j = urlopen("http://people.canonical.com/~ubuntu-archive/package-team-mapping.json")
+    else:
+        j = open(subscribers_json, 'rb')
+    with j:
+        team_to_packages = json.loads(j.read().decode('utf-8'))
+    package_to_teams = {}
+    for team, team_packages in team_to_packages.items():
+        for package in team_packages:
+            if package in packages:
+                package_to_teams.setdefault(package, []).append(team)
+    return package_to_teams
+
+def packages_with_bzr_repos(options):
+    pkgs = {}
+    for component in options.components.split(','):
+        sources_path = "%s/dists/%s/%s/source/Sources.gz" % (
+            options.archive_dir, options.suite, component)
+        for section in apt_pkg.TagFile(decompress_open(sources_path)):
+            if 'Package' in section and 'Vcs-Bzr' in section:
+                pkgs[section['Package']] = section['Vcs-Bzr']
+    return pkgs
+
+
+def repo_url(repo):
+    return re.sub('^lp:','https://code.launchpad.net/',repo)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--archive-dir', action='store', default=os.path.expanduser("~/mirror/ubuntu"))
+    parser.add_argument('--components', action='store', default="main,restricted,universe,multiverse")
+    parser.add_argument('--suite', action='store')
+    parser.add_argument('--subscribers-json', action='store')
+    parser.add_argument('output')
+    args = parser.parse_args()
+
+    pkgs = packages_with_bzr_repos(args)
+
+    print("getting subscribers")
+    subscribers = get_subscribers_json(set(pkgs.keys()), args.subscribers_json)
+    for p in set(pkgs.keys()):
+        if p not in subscribers:
+            subscribers[p] = ['unknown']
+
+    all_teams = set()
+    team_to_pkgs = defaultdict(list)
+    for package, teams in subscribers.items():
+        all_teams |= set(teams)
+        for team in teams:
+            team_to_pkgs[team].append(package)
+
+    team_to_attn_count = {}
+    for team, packages in team_to_pkgs.items():
+        team_to_attn_count[team] = len(packages)
+        team_to_pkgs[team] = [(package, repo_url(pkgs[package])) for package in sorted(packages)]
+
+    print("rendering")
+    t = env.get_template('team-report-bzr.html')
+    now = datetime.utcnow()
+    with open(args.output, 'w', encoding='utf-8') as fp:
+        fp.write(t.render(
+            all_teams=all_teams,
+            now="%s.%s.%s %d:%s:%s UTC" % (now.year, now.month, now.day, now.hour, now.minute, now.second),
+            team_to_pkgs=team_to_pkgs,
+            team_to_attn_count=team_to_attn_count))
+
+if __name__ == '__main__':
+    main()
 
=== added file 'templates/team-report-bzr.html'
--- templates/team-report-bzr.html	1970-01-01 00:00:00 +0000
+++ templates/team-report-bzr.html	2018-11-23 17:03:38 +0000
@@ -0,0 +1,38 @@
+<!DOCTYPE HTML>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <title>Bazaar packaging repositories by team</title>
+    <style>
+      body { font-family: ubuntu }
+      .not-late { color: grey; }
+      .not-late a { color: #77f; }
+    </style>
+  </head>
+  <body lang="en">
+    <h1>Bazaar packaging repositories by team</h1>
+    <p>Git migration tools and schedule: <a href="https://wiki.ubuntu.com/UbuntuDevelopment/MigratingFromBzrToGit">https://wiki.ubuntu.com/UbuntuDevelopment/MigratingFromBzrToGit</a></p
+    <p>
+      Generated: {{ now }}
+    <ul>
+      {% for team in all_teams|sort %}
+      <li><a href="#{{ team }}">{{ team }}</a> ({{ ngettext("%(num)d packaging repository", "%(num)d packaging repositories", team_to_attn_count[team]) }})</li>
+      {% endfor %}
+    </ul>
+    {% for team in all_teams|sort %}
+    <h1 id="{{ team }}">{{ team }}</h1>
+    <p>
+      {{ ngettext("%(num)d packaging repositories", "%(num)d packaging repositories", team_to_attn_count[team]) }}
+    </p>
+    <ul>
+      {% for pkg in team_to_pkgs[team] %}
+      {% set p = pkg[0] %}
+      {% set url = pkg[1] %}
+      <li>
+        <b><a href="https://launchpad.net/ubuntu/+source/{{ p }}/">{{ p }}</a>:</b> <a href="{{ url }}">{{ url }}</a>
+      </li>
+      {% endfor %}
+    </ul>
+    {% endfor %}
+  </body>
+</html>