Merge lp:~jelmer/lptools/recipe-status into lp:lptools

[Jelmer Vernooij]

Import "recipe-status".

This script generates a text or html overview of the status of all recipes owned by a particular person or team.

For example output, see http://people.canonical.com/~jelmer/recipe-status/bzr.html

I had to add a new utility function data_dir, which tries to find the templates for recipe-status. It's a bit hackish, so I'd love to hear suggestions for better ways to do it.

[lp:~jelmer/lptools/recipe-status]

28. By Jelmer Vernooij on 2011-08-27

Fix some long lines.

29. By Jelmer Vernooij on 2011-08-30

Rename to lp-recipe-status, exclude distroseries that have been disabled.

[Martin Pool (mbp)]

The main function is not the prettiest, but overall it looks decent and worth having in here. You ought to mention it in the readme, and maybe on the launchpad blog.

review: Approve

[Jelmer Vernooij (jelmer)]

I've factored out the two main bits of the main function so it should be a bit more readable now, and added a bunch of docstrings.

=== added file 'bin/lp-recipe-status'
--- bin/lp-recipe-status	1970-01-01 00:00:00 +0000
+++ bin/lp-recipe-status	2011-08-30 14:19:50 +0000
@@ -0,0 +1,210 @@
+#!/usr/bin/python
+
+# Copyright (C) 2011 Jelmer Vernooij <[e-mail]>
+#
+# ##################################################################
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# See file /usr/share/common-licenses/GPL-3 for more details.
+#
+# ##################################################################
+#
+# vi: expandtab:sts=4
+
+"""Show the status of the recipes owned by a particular user.
+"""
+
+from cStringIO import StringIO
+import gzip
+import optparse
+import os
+import re
+import sys
+import urllib
+
+from lptools import config
+
+try:
+    import tdb
+except ImportError:
+    cache = {}
+else:
+    cache = tdb.Tdb("recipe-status-cache.tdb", 1000, tdb.DEFAULT,
+        os.O_RDWR|os.O_CREAT)
+
+def gather_per_distroseries_source_builds(recipe):
+    last_per_distroseries = {}
+    for build in recipe.completed_builds:
+        if build.distro_series.self_link not in recipe.distroseries:
+            # Skip distro series that are no longer being build
+            continue
+        distro_series_name = build.distro_series.name
+        previous_build = last_per_distroseries.get(distro_series_name)
+        if previous_build is None or previous_build.datecreated < build.datecreated:
+            last_per_distroseries[distro_series_name] = build
+    return last_per_distroseries
+
+
+def build_failure_link(build):
+    if build.buildstate == "Failed to upload":
+        return build.upload_log_url
+    elif build.buildstate in ("Failed to build", "Dependency wait", "Chroot problem"):
+        return build.build_log_url
+    else:
+        return None
+
+version_matcher = re.compile("^dpkg-buildpackage: source version (.*)$")
+source_name_matcher = re.compile("^dpkg-buildpackage: source package (.*)$")
+
+def source_build_find_version(source_build):
+    cached_version = cache.get("version/%s" % str(source_build.self_link))
+    if cached_version:
+        return tuple(cached_version.split(" "))
+    # FIXME: Find a more efficient way to retrieve the package/version that was built
+    build_log_gz = urllib.urlopen(source_build.build_log_url)
+    build_log = gzip.GzipFile(fileobj=StringIO(build_log_gz.read()))
+    version = None
+    source_name = None
+    for l in build_log.readlines():
+        m = version_matcher.match(l)
+        if m:
+            version = m.group(1)
+        m = source_name_matcher.match(l)
+        if m:
+            source_name = m.group(1)
+    if not source_name:
+        raise Exception("unable to find source name in %s" %
+            source_build.build_log_url)
+    if not version:
+        raise Exception("unable to find version in %s" %
+            source_build.build_log_url)
+    cache["version/%s" % str(source_build.self_link)] = "%s %s" % (
+        source_name, version)
+    return (source_name, version)
+
+
+def find_binary_builds(recipe, source_builds):
+    ret = {}
+    for source_build in source_builds:
+        archive = source_build.archive
+        (source_name, version) = source_build_find_version(source_build)
+        sources = archive.getPublishedSources(
+            distro_series=source_build.distro_series,
+            exact_match=True, pocket="Release", source_name=source_name,
+            version=version)
+        assert len(sources) == 1
+        source = sources[0]
+        ret[source_build.distro_series.name] = source.getBuilds()
+    return ret
+
+
+def build_failure_summary(build):
+    # FIXME: Perhaps parse the relevant logs and extract a summary line?
+    return build.buildstate
+
+def build_class(build):
+    return {
+        "Failed to build": "failed-to-build",
+        "Failed to upload": "failed-to-upload",
+        "Dependency wait": "dependency-wait",
+        "Chroot problem": "chroot-problem",
+        "Uploading build": "uploading-build",
+        "Currently building": "currently-building",
+        "Build for superseded Source": "superseded-source",
+        "Successfully built": "successfully-built",
+        "Needs building": "needs-building",
+        }[build.buildstate]
+
+def filter_source_builds(builds):
+    sp_success = set()
+    sp_failures = set()
+    for build in builds:
+        if build.buildstate == "Successfully built":
+            sp_success.add(build)
+        else:
+            sp_failures.add(build)
+    return (sp_success, sp_failures)
+
+
+def main(argv):
+    parser = optparse.OptionParser('%prog [options] PERSON\n\n'
+        '  PERSON is the launchpad person whose recipes to check')
+    parser.add_option("--html", help="Generate HTML", action="store_true")
+    opts, args = parser.parse_args()
+    if len(args) != 1:
+        parser.print_usage()
+        return 1
+    person = args[0]
+    launchpad = config.get_launchpad("recipe-status")
+    person = launchpad.people[person]
+
+    if opts.html:
+        from chameleon.zpt.loader import TemplateLoader
+        tl = TemplateLoader(os.path.join(config.data_dir(), "templates"))
+        relevant_distroseries = set()
+        source_builds = {}
+        binary_builds = {}
+        all_binary_builds_ok = {}
+        for recipe in person.recipes:
+            sys.stderr.write("Processing recipe %s\n" % recipe.name)
+            last_per_distroseries = gather_per_distroseries_source_builds(recipe)
+            source_builds[recipe.name] = last_per_distroseries
+            relevant_distroseries.update(set(last_per_distroseries))
+            (sp_success, sp_failures) = filter_source_builds(last_per_distroseries.values())
+            binary_builds[recipe.name] = find_binary_builds(recipe, sp_success)
+            all_binary_builds_ok[recipe.name] = {}
+            for distroseries, recipe_binary_builds in binary_builds[recipe.name].iteritems():
+                all_binary_builds_ok[recipe.name][distroseries] = all(
+                    [bb.buildstate == "Successfully built" for bb in recipe_binary_builds])
+        relevant_distroseries = list(relevant_distroseries)
+        relevant_distroseries.sort()
+        page = tl.load("recipe-status.html")
+        print page.render(person=person,
+            relevant_distroseries=relevant_distroseries,
+            recipes=person.recipes, source_builds=source_builds,
+            build_failure_summary=build_failure_summary,
+            build_failure_link=build_failure_link,
+            binary_builds=binary_builds,
+            ubuntu=launchpad.distributions["ubuntu"],
+            build_class=build_class,
+            all_binary_builds_ok=all_binary_builds_ok)
+    else: # text
+        for recipe in person.recipes:
+            last_per_distroseries = gather_per_distroseries_source_builds(recipe)
+            (sp_success, sp_failures) = filter_source_builds(last_per_distroseries.values())
+            sp_success_distroseries = [build.distro_series.name for build in sp_success]
+            if sp_failures:
+                print "%s source build failures (%s successful):" % (recipe.name, ", ".join(sp_success_distroseries))
+                for failed_build in sp_failures:
+                    url = build_failure_link(failed_build)
+                    sys.stdout.write("  %s(%s)" % (failed_build.distro_series.name, failed_build.buildstate))
+                    if url:
+                        sys.stdout.write(": %s" % url)
+                    sys.stdout.write("\n")
+            elif sp_success:
+                print "%s source built successfully on %s" % (recipe.name, ", ".join(sp_success_distroseries))
+            else:
+                print "%s never built" % recipe.name
+            binary_builds = find_binary_builds(recipe, sp_success)
+            for source_build in sp_success:
+                for binary_build in binary_builds[source_build]:
+                    if binary_build.buildstate != "Successfully built":
+                        url = build_failure_link(binary_build)
+                        sys.stdout.write("  %s,%s(%s)" %
+                                (binary_build.distro_series.name,
+                                    binary_build.arch_tag,
+                                    binary_build.buildstate))
+                        if url:
+                            sys.stdout.write(": %s" % url)
+                        sys.stdout.write("\n")
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

=== modified file 'lptools/config.py'
--- lptools/config.py	2011-08-29 14:21:32 +0000
+++ lptools/config.py	2011-08-30 14:19:50 +0000
@@ -19,6 +19,7 @@
 """Configuration glue for lptools."""
 
 __all__ = [
+    "data_dir",
     "ensure_dir",
     "get_launchpad",
     ]
@@ -44,3 +45,15 @@
         cachedirs.
     """
     return Launchpad.login_with("lptools-%s" % appname, "production")
+
+
+def data_dir():
+    """Return the arch-independent data directory.
+    """
+    # Running from source directory?
+    ret = os.path.join(os.path.dirname(__file__), "..")
+    if os.path.exists(os.path.join(ret, "templates")):
+        return ret
+    else:
+        return os.path.abspath(os.path.join(os.path.dirname(__file__),
+            "../../../../share/lptools"))

=== modified file 'setup.py'
--- setup.py	2010-04-23 15:16:45 +0000
+++ setup.py	2011-08-30 14:19:50 +0000
@@ -16,6 +16,8 @@
     description='A collection of tools for developers who use launchpad',
     long_description=description,
     py_modules=[],
+    data_files=[('share/lptools/templates', ['templates/recipe-status.css',
+                                             'templates/recipe-status.html'])],
     packages=['lptools'],
     scripts=glob('bin/*'),
     classifiers = [

=== added directory 'templates'
=== added file 'templates/recipe-status.css'
--- templates/recipe-status.css	1970-01-01 00:00:00 +0000
+++ templates/recipe-status.css	2011-08-30 14:19:50 +0000
@@ -0,0 +1,38 @@
+body {
+    background: url(background.png) repeat;
+    color: #222222;
+    font-family: "UbuntuBeta", Ubuntu, "Bitstream Vera Sans", "DejaVu Sans", Tahoma, sans-serif;
+    font-size: 12px;
+}
+
+a {
+    text-decoration: none;
+}
+
+a:hover {
+    text-decoration: underline;
+}
+
+a:visited {
+    color: blue;
+}
+
+.failed-to-upload, .failed-to-build {
+	background: red;
+}
+
+.chroot-problem, .dependency-wait {
+	background: orange;
+}
+
+.not-available {
+	background: gray;
+}
+
+.successfully-built {
+	background: chartreuse;
+}
+
+.partial-success {
+	background: yellow;
+}

=== added file 'templates/recipe-status.html'
--- templates/recipe-status.html	1970-01-01 00:00:00 +0000
+++ templates/recipe-status.html	2011-08-30 14:19:50 +0000
@@ -0,0 +1,42 @@
+<html>
+  <head>
+    <title>Recipe status for <omit tal:replace="person.name"/></title>
+    <link rel="stylesheet" type="text/css" href="recipe-status.css" />
+  </head>
+
+  <body>
+    <table class="recipes">
+      <tr>
+        <th>Recipe</th>
+        <th tal:repeat="distroseries relevant_distroseries" class="recipe-column">
+          <a tal:attributes="href ubuntu.getSeries(name_or_version=distroseries).web_link" tal:content="distroseries"/>
+        </th>
+      </tr>
+      <tr tal:repeat="recipe recipes" class="recipe">
+        <td class="recipe"><a tal:attributes="href recipe.web_link" tal:content="recipe.name"/></td>
+        <distroseries tal:omit-tag="" tal:repeat="distroseries relevant_distroseries">
+          <source-build tal:omit-tag="" tal:define="global source_build source_builds[recipe.name].get(distroseries)"/>
+          <td tal:condition="not source_build" class="not-available">N/A</td>
+          <td tal:condition="source_build and source_build.buildstate != 'Successfully built'" tal:attributes="class build_class(source_build)">
+            <a tal:attributes="href build_failure_link(source_build)"
+              tal:omit-tag="not build_failure_link(source_build)"
+              tal:content="'Source:'+build_failure_summary(source_build)"/>
+          </td>
+          <source-build-success tal:omit-tag="" tal:condition="source_build and source_build.buildstate == 'Successfully built'">
+            <td tal:condition="all_binary_builds_ok[recipe.name].get(distroseries)" class="successfully-built">OK</td>
+            <td tal:condition="not all_binary_builds_ok[recipe.name].get(distroseries)" class="partial-success">
+              <binary-build tal:omit-tag="" tal:repeat="binary_build binary_builds[recipe.name][distroseries]">
+              <div tal:attributes="class build_class(binary_build)">
+                <a tal:attributes="href build_failure_link(binary_build)"
+                  tal:omit-tag="not build_failure_link(binary_build)"
+                  tal:content="binary_build.arch_tag+':'+build_failure_summary(binary_build)"/>
+                </div>
+              </binary-build>
+            </td>
+          </source-build-success>
+        </distroseries>
+      </tr>
+    </table>
+    <p>Generated using recipe-status. For the source see <a href="https://code.launchpad.net/lptools">lptools</a>.</p>
+  </body>
+</html>