Merge lp:~work-items-tracker-hackers/launchpad-work-items-tracker/blueprints-api into lp:launchpad-work-items-tracker

[Martin Pitt]

This branch contains lp:~james-w/launchpad-work-items-tracker/blueprints-api, lp:~james-w/launchpad-work-items-tracker/templates, and the other excellent work from James. It requires installing python-mako on people.canonical.com before I can merge this into trunk and roll this out; RT is pending.

[Martin Pitt (pitti)]

review: Approve

[James Westby (james-w)]

Oh yeah, I forgot mako wasn't on people, sorry. I would have mentioned
this otherwise.

Thanks,

James

[Martin Pitt (pitti)]

It's installed now, merged. Thanks James!

=== added file '.bzrignore'
--- .bzrignore	1970-01-01 00:00:00 +0000
+++ .bzrignore	2011-02-09 12:26:14 +0000
@@ -0,0 +1,1 @@
+.testrepository

=== added file '.testr.conf'
--- .testr.conf	1970-01-01 00:00:00 +0000
+++ .testr.conf	2011-02-09 12:26:14 +0000
@@ -0,0 +1,3 @@
+[DEFAULT]
+test_command=python -m subunit.run $IDLIST
+test_id_list_default=tests.test_suite

=== modified file 'collect'
--- collect	2011-01-29 09:08:54 +0000
+++ collect	2011-02-09 12:26:14 +0000
@@ -3,11 +3,11 @@
 # Pull work items from various sources (blueprint whiteboards, linked blueprint
 # bugs, wiki pages) and put them into a database.
 
-import urllib, re, sys, optparse, smtplib, pwd, os
+import urllib, re, sys, optparse, smtplib, pwd, os, urlparse
 from email.mime.text import MIMEText
 import sqlite3 as dbapi2
 
-from launchpadlib.launchpad import Launchpad, EDGE_SERVICE_ROOT
+from launchpadlib.launchpad import Launchpad
 
 import report_tools
 
@@ -59,51 +59,20 @@
         dbg('Queueing data error: ' + s)
 
 
+def web_link(item):
+    """Get a link to the Launchpad API object on the website."""
+    api_link = item.self_link
+    parts = urlparse.urlparse(api_link)
+    return parts.scheme + "://" + parts.netloc.replace("api.", "") + "/" + parts.path.split("/", 2)[2]
+
+
 ########################################################################
 #
 # Functions for parsing Launchpad data
 #
 ########################################################################
 
-def lp_blueprints_from_list(url, name_pattern = None):
-    '''Return blueprints from a LP list view.
-
-    This can optionally specify a name pattern (which is mostly useful for
-    testing and debugging, to only parse a small subset of blueprints)
-
-    Return a dictionary name -> (url, milestone).
-    '''
-    blueprint_name_filter = re.compile('href="(/[a-zA-Z0-9-]+/\+spec/%s[^"]+)"(?!.*class.*sprite)' %
-            (name_pattern or '.'))
-    milestone_re = re.compile('href=".*/[a-zA-Z0-9-]+/\+milestone/([^"]+)"')
-
-    result = {}
-    scan_tr_end = False
-    milestone = None
-    bp = None
-    dbg("lp_blueprints_from_list(): Loading '%s'" % url)
-    for l in urllib.urlopen(url):
-        if scan_tr_end:
-            m = milestone_re.search(l)
-            if m:
-                milestone = m.group(1)
-                dbg('lp_blueprints_from_list(): current spec milestone: ' + milestone)
-            if '</tr>' in l:
-                scan_tr_end = False
-                if bp:
-                    result[bp.split('/')[-1]] = (bp, milestone)
-                bp = None
-                milestone = None
-        else:
-            m = blueprint_name_filter.search(l)
-            if m:
-                bp = report_tools.blueprints_base_url + m.group(1)
-                dbg('lp_blueprints_from_list(): found BP: ' + bp)
-                scan_tr_end = True
-
-    return result
-
-def lp_import_blueprint(db, name, url, contents):
+def lp_import_blueprint(db, bp):
     '''Import details of a blueprint into the specs table.
 
     If a blueprint does not have a status, use the description.
@@ -117,56 +86,50 @@
     cur.execute('SELECT name FROM milestones')
     valid_milestones = [m[0] for m in cur] + [None]
 
-    queries = {
-        'priority': '<dt>Priority:</dt>\s*<dd>\s*([^\n]*)',
-        'definition': '<dt>Definition:</dt>\s*<dd>\s*([^\n]*)',
-        'implementation': '<dt>Implementation:</dt>\s*<dd>\s*([^\n]*)',
-        'approver': '<dt>Approver:</dt>\s*<dd>\s*<a href="https://.*?launchpad.net/~([a-zA-Z0-9_-]+)" ',
-        'drafter': '<dt>Drafter:</dt>\s*<dd>\s*<a href="https://.*?launchpad.net/~([a-zA-Z0-9_-]+)" ',
-        'assignee': '<dt>Assignee:</dt>\s*<dd>\s*<a href="https://.*?launchpad.net/~([a-zA-Z0-9_-]+)" ',
-        'milestone': '<dt>Milestone target:</dt>\s*<dd>\s*<a href="https://.*?launchpad.net/\w+/\+milestone/([a-zA-Z0-9_\.-]+)"',
-        'description': '<div class="top-portlet">\s*<p>(.*?)</p>',
-        'status': '(?:<p>|^)[Ss]tatus:\s*<br />(.*?)</p>',
-        'details_url': 'href="([^"]*)">Read the full specification</a>',
-        'roadmap_notes': '(?:<p>|^)[Rr]oadmap\s+[Nn]otes:\s*<br />(.*?)</p>',
-    }
+    def get_whiteboard_section(heading):
+        if bp.whiteboard is None:
+            return None
+        regex = '^' + heading + ':\s*\n(.*?)\n\n'
+        m = re.search(regex, bp.whiteboard, re.S | re.I | re.M)
+        if m is not None:
+            section = m.group(1).strip()
+        else:
+            section = None
+        return section
 
     data = {}
-    for key, r in queries.iteritems():
-        m = re.search(r, contents, re.S)
-        if not m:
-            dbg('lp_import_blueprint(%s): did not find a match for key %s' % (name, key))
-            if key == 'description':
-                data[key] = '(no description)'
-            else:
-                data[key] = None
-        else:
-            data[key] = m.group(1).strip()
+    data['priority'] = bp.priority
+    data['definition'] = bp.definition_status
+    data['implementation'] = bp.implementation_status
+    data['approver'] = bp.approver and bp.approver.name or None
+    data['drafter'] = bp.drafter and bp.drafter.name or None
+    data['assignee'] = bp.assignee and bp.assignee.name or None
+    data['milestone'] = bp.milestone and bp.milestone.name or None
+    data['description'] = bp.summary or "(no description)"
+    data['status'] = get_whiteboard_section('Status')
+    data['details_url'] = bp.specification_url
+    data['roadmap_notes'] = get_whiteboard_section('Roadmap\s+Notes')
 
     # ignore "later" milestone"
     if data['milestone'] == 'later':
-        dbg('lp_import_blueprint(%s): spec has "later" milestone, ignoring it' % name)
+        dbg('lp_import_blueprint(%s): spec has "later" milestone, ignoring it' % bp.name)
         return False
 
     if data['milestone'] not in valid_milestones:
-        data_error(url, 'milestone "%s" is unknown/invalid' % data['milestone'], True)
+        data_error(web_link(bp), 'milestone "%s" is unknown/invalid' % data['milestone'], True)
 
     # if we have an explicit status: in the whiteboard, use that; otherwise use
     # description as status text
     if data['status']:
-        data['status'] = data['status'].replace('<br />',
-                '\n').replace('</div>', '').replace('&nbsp;', ' ').replace(
-                '<wbr></wbr>', '').strip()
+        data['status'] = data['status'].strip()
     else:
-        data['status'] = data['description'].replace('<br />',
-                '\n').replace('</div>', '').replace('&nbsp;', ' ').replace(
-                '<wbr></wbr>', '').strip()
+        data['status'] = data['description'].strip()
     del data['description']
 
-    dbg('lp_import_blueprint(%s): finished parsing; data: %s' % (name, str(data)))
+    dbg('lp_import_blueprint(%s): finished parsing; data: %s' % (bp.name, str(data)))
 
     db.cursor().execute('INSERT INTO specs VALUES (?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?,?)',
-            (name, url, data['priority'], data['implementation'],
+            (bp.name, web_link(bp), data['priority'], data['implementation'],
             data['assignee'], data['status'], data['milestone'],
             data['definition'], data['drafter'], data['approver'],
             data['details_url'], data['roadmap_notes']))
@@ -178,10 +141,7 @@
 
     '''
 
-    ### cargo culted from parse_blueprint_workitem
-    line = line.replace('<br />', '').replace('</div>', '').replace('</p>',
-            '').replace('<wbr></wbr>', '').replace('&nbsp;', ' ').replace(
-            '<a href="/', '<a href="https://launchpad.net/').strip()
+    line = line.strip()
     if not line:
         return
 
@@ -201,14 +161,9 @@
     db.commit()
 
 def parse_complexity_item(lp, db, line, bp_name, bp_url, def_milestone, def_assignee):
-    
-    line = line.replace('<br />', '').replace('</div>', '').replace('</p>',
-            '').replace('<wbr></wbr>', '').replace('&nbsp;', ' ').replace(
-            '<a href="/', '<a href="https://launchpad.net/').strip()
-            
-    # remove special characters people tend to type            
+    line = line.strip()
+    # remove special characters people tend to type
     line = re.sub('[^\w -.]', '', line)
-                
     if not line:
         return
 
@@ -219,17 +174,15 @@
     assignee    = None
 
     try:
-        list = line.split()
-        list.reverse()
+        complexity_list = line.split()
+        complexity_list.reverse()
 
         # we may not have any values in the list, so append our
         # default values in the right order so they can be mapped
         defs = [None, def_milestone, def_assignee]
-        for i in range(len(list), len(defs)):
-            list.append(defs[i])
-            
-        (num, milestone, assignee) = list
-    
+        for i in range(len(complexity_list), len(defs)):
+            complexity_list.append(defs[i])
+        (num, milestone, assignee) = complexity_list
         if not num:
             data_error(bp_url, 'No complexity points defined for %s' % line)
 
@@ -238,17 +191,17 @@
         cur = db.cursor()
         cur.execute('INSERT INTO complexity VALUES(?,?,?,?,date(CURRENT_TIMESTAMP))',
                 (assignee, num, milestone, bp_name))
-
     except ValueError:
         data_error(bp_url, "\tComplexity line '%s' could not be parsed %s" % (line, ValueError))
 
+
 def parse_blueprint_workitem(line, default_assignee, milestone,
         blueprint_url, launchpad, result_list):
     '''Parse a work item line of a blueprint whiteboard.'''
 
-    line = line.replace('<br />', '').replace('</div>', '').replace('</p>',
-            '').replace('<wbr></wbr>', '').replace('&nbsp;', ' ').replace(
-            '<a href="/', '<a href="https://launchpad.net/').strip()
+    # FIXME: we lose bug linking etc. that is done by the tales
+    # formatters in LP here.
+    line = line.strip()
     if not line:
         return
     dbg("\tworkitem (clean): '%s'" % (line))
@@ -291,13 +244,10 @@
 
     result_list.append((desc, state, assignee, milestone))
 
-def follow_blueprint_buglink(bugnum, default_assignee, blueprint_name,
+def follow_blueprint_buglink(bug, default_assignee, blueprint_name,
         launchpad, release, default_milestone, result_list):
     '''Query launchpad for the information on a linked bug'''
 
-    bugnum = int(bugnum)
-    dbg('follow_blueprint_buglink(): processing bug %i (default milestone: %s)' % (bugnum, default_milestone))
-    bug = launchpad.bugs[bugnum]
     for task in bug.bug_tasks:
         if task.status == 'Unknown':
             # this can happen for upstream tasks
@@ -338,7 +288,7 @@
             if better_task:
                 continue
 
-        desc = '<a href="https://launchpad.net/bugs/%d">LP: #%d</a>: ' % (bugnum, bugnum) + bug.title
+        desc = '<a href="https://launchpad.net/bugs/%d">LP: #%d</a>: ' % (bug.id, bug.id) + bug.title
         if rtype != 'distribution':
             desc += ' (%s)' % target.name
 
@@ -363,32 +313,27 @@
             return word
     return None
 
-def lp_import_blueprint_workitems(lp, db, bp_name, bp_url, contents, release):
+def lp_import_blueprint_workitems(lp, db, bp, release):
     '''Collect work items from a Launchpad blueprint.
 
     This includes work items from the whiteboard as well as linked bugs.
     '''
-    linked_bugs_re = re.compile('<div id="bug_links".*>', re.I)
-    bugnum_re = re.compile('<a href="https://bugs\..*launchpad\.net/bugs/([0-9]+)" class="sprite bug">')
-    work_items_re = re.compile('(<p>|^)work items(.*)\s*:\s*<br />', re.I)
-    meta_re = re.compile( '(<p>|^)Meta.*?:<br />', re.I )
-    complexity_re = re.compile( '(<p>|^)Complexity.*?:<br />', re.I )
+    work_items_re = re.compile('^work items(.*)\s*:\s*$', re.I)
+    meta_re = re.compile('^Meta.*?:$', re.I)
+    complexity_re = re.compile('^Complexity.*?:$', re.I)
 
     in_workitems_block = False
-    in_linked_bugs_block = False
     in_meta_block = False
     in_complexity_block = False
-    found_linked_bugs = False
     work_items = []
-    found_wb_workitems = False
     milestone = None
 
     cur = db.cursor()
-    cur.execute('SELECT assignee, milestone, implementation FROM specs WHERE name = ?', (bp_name,))
+    cur.execute('SELECT assignee, milestone, implementation FROM specs WHERE name = ?', (bp.name,))
     (spec_assignee, spec_milestone, spec_implementation) = cur.fetchone()
 
     dbg('lp_import_blueprint_workitems(): processing %s (spec milestone: %s, spec assignee: %s, spec implementation: %s)' % (
-        bp_name, spec_milestone, spec_assignee, spec_implementation))
+        bp.name, spec_milestone, spec_assignee, spec_implementation))
 
     cur.execute('SELECT team FROM teams WHERE name = ?', (spec_assignee,))
     assignee_teams = [t[0] for t in cur]
@@ -396,69 +341,51 @@
     cur.execute('SELECT name FROM milestones')
     valid_milestones = [m[0] for m in cur]
 
-    for l in contents.splitlines():
-
-        if not in_workitems_block and not in_linked_bugs_block \
-                and not in_meta_block and not in_complexity_block:
-            m = work_items_re.search(l)
-            if m:
-                in_workitems_block = True
-                found_wb_workitems = True
-                dbg('lp_import_blueprint_workitems(): starting work items block at ' + l)
-                if not milestone:
-                    milestone = milestone_extract(m.group(2), valid_milestones)
-                    dbg('  ... setting milestone to ' + str(milestone))
-            if linked_bugs_re.search(l):
-                in_linked_bugs_block = True
-                found_linked_bugs = True
-            if meta_re.search(l):
-                in_meta_block = True
-            if complexity_re.search(l):
-                in_complexity_block = True                
-            continue
-
-        if in_workitems_block:
-            dbg("\tworkitem (raw): '%s'" % (l.strip()))
-            parse_blueprint_workitem(l, spec_assignee, milestone or
-                    spec_milestone, bp_url, lp, work_items)
-
-            if '</p>' in l:
-                dbg('lp_import_blueprint_workitems(): closing work items block with line: ' + l)
-                in_workitems_block = False
-                milestone = None
-
-        if in_linked_bugs_block:
-            dbg("\tbug block line (raw): '%s'" % (l.strip()))
-            m = bugnum_re.search(l)
-            if m and lp:
-                follow_blueprint_buglink(m.group(1), spec_assignee,
-                        bp_name, lp, release,
-                        milestone or spec_milestone, work_items)
-
-            if '</div>' in l:
-                in_linked_bugs_block = False
-                continue
-
-        if in_meta_block:
-            dbg("\tmeta line (raw): '%s'" % (l.strip()))
-            parse_meta_item( lp, db, l, bp_name )
-
-            # done with the meta information?
-            if '</p>' in l:
-                in_meta_block = False
-                continue
-
-        if in_complexity_block:
-            dbg("\tcomplexity block line (raw): '%s'" % (l.strip()))
-            parse_complexity_item(lp, db, l, bp_name, bp_url, spec_milestone, spec_assignee)
-            
-            # done with the meta information?
-            if '</p>' in l:
-                in_complexity_block = False
-                continue
-
-        if found_wb_workitems and '</div>' in l:
-            break
+    if bp.whiteboard:
+        for l in bp.whiteboard.splitlines():
+
+            if (not in_workitems_block
+                and not in_meta_block and not in_complexity_block):
+                m = work_items_re.search(l)
+                if m:
+                    in_workitems_block = True
+                    dbg('lp_import_blueprint_workitems(): starting work items block at ' + l)
+                    if not milestone:
+                        milestone = milestone_extract(m.group(1), valid_milestones)
+                        dbg('  ... setting milestone to ' + str(milestone))
+                if meta_re.search(l):
+                    in_meta_block = True
+                if complexity_re.search(l):
+                    in_complexity_block = True
+                continue
+
+            if in_workitems_block:
+                dbg("\tworkitem (raw): '%s'" % (l.strip()))
+                if not l.strip():
+                    dbg('lp_import_blueprint_workitems(): closing work items block with line: ' + l)
+                    in_workitems_block = False
+                    milestone = None
+                parse_blueprint_workitem(l, spec_assignee, milestone or
+                        spec_milestone, web_link(bp), lp, work_items)
+
+            if in_meta_block:
+                dbg("\tmeta line (raw): '%s'" % (l.strip()))
+                if not l.strip():
+                    in_meta_block = False
+                    continue
+                parse_meta_item(lp, db, l, bp.name)
+
+            if in_complexity_block:
+                dbg("\tcomplexity block line (raw): '%s'" % (l.strip()))
+                if not l.strip():
+                    in_complexity_block = False
+                    continue
+                parse_complexity_item(lp, db, l, bp.name, web_link(bp), spec_milestone, spec_assignee)
+
+    if lp:
+        for bug in bp.bugs:
+            follow_blueprint_buglink(bug, spec_assignee, bp.name, lp, release,
+                    spec_milestone, work_items)
 
     if not work_items:
         #data_error(bp_url, 'no work items defined', True)
@@ -474,7 +401,8 @@
 
     for (desc, status, assignee, milestone) in work_items:
         db.cursor().execute('INSERT INTO work_items VALUES (?, ?, ?, ?, ?, date(CURRENT_TIMESTAMP))',
-                (desc, bp_name, status, assignee, milestone))
+                (desc, bp.name, status, assignee, milestone))
+
 
 def lp_import_milestones(lp_project, db):
     '''Import milestones from Launchpad into DB.
@@ -550,29 +478,41 @@
         db.cursor().execute('INSERT INTO work_items VALUES (?, ?, ?, ?, ?, date(CURRENT_TIMESTAMP))',
                 (title, name, state, assignee, milestone))
 
-def lp_import(db, cfg, name_pattern = None):
+def lp_import(db, cfg, name_pattern=None):
     '''Collect blueprint work items and status from Launchpad into DB.'''
 
-    lp = Launchpad.login_with('ubuntu-work-items', service_root=EDGE_SERVICE_ROOT)
+    lp = Launchpad.login_with('ubuntu-work-items', service_root="production", version="devel")
+    projects = []
 
     if 'release' in cfg:
         lp_project = lp.distributions['ubuntu'].getSeries(name_or_version=cfg['release'])
-        url = '%s/ubuntu/%s/+specs?batch=300' % (
-                report_tools.blueprints_base_url, cfg['release'])
+        projects.append(lp_project)
     else:
         assert 'project' in cfg, 'Configuration needs to specify project or release'
         lp_project = lp.projects[cfg['project']]
-        url = '%s/%s/+specs?batch=300' % (
-                report_tools.blueprints_base_url, cfg['project'])
+        projects.append(lp_project)
 
     lp_import_milestones(lp_project, db)
     lp_import_teams(lp, db, cfg)
 
-    for (bp, (url, milestone)) in lp_blueprints_from_list(url, name_pattern).iteritems():
-        dbg('lp_import(): downloading %s from %s' % (bp, url))
-        contents = urllib.urlopen(url).read().decode('UTF-8')
-        if lp_import_blueprint(db, bp, url, contents):
-            lp_import_blueprint_workitems(lp, db, bp, url, contents, cfg.get('release'))
+    extra_projects = cfg.get('extra_projects', [])
+    for extra_project_name in extra_projects:
+        extra_project = lp.projects[extra_project_name]
+        lp_import_milestones(extra_project, db)
+        projects.append(extra_project)
+
+    if name_pattern:
+        name_filter = re.compile(name_pattern)
+
+    for project in projects:
+        # XXX: should this be valid_ or all_specifications?
+        for bp in project.valid_specifications:
+            if name_pattern:
+                if not name_filter.search(bp.name):
+                    continue
+            dbg('lp_import(): downloading %s from %s' % (bp.name, bp.self_link))
+            if lp_import_blueprint(db, bp):
+                lp_import_blueprint_workitems(lp, db, bp, cfg.get('release'))
 
     lp_import_bug_workitems(lp_project, db, cfg)
 
@@ -944,6 +884,12 @@
     if 'bug_status_map' in cfg:
         bug_wi_states.update(cfg['bug_status_map'])
 
+    lock_path = opts.database + ".collect_lock"
+    lock_f = open(lock_path, "wb")
+    if report_tools.lock_file(lock_f) is None:
+        print "Another instance is already running"
+        sys.exit(0)
+
     db = get_db(opts.database)
 
     # reset status for current day
@@ -962,3 +908,4 @@
     db.commit()
     send_error_mails(cfg)
 
+    os.unlink(lock_path)

=== modified file 'generate-all'
--- generate-all	2010-09-14 15:49:33 +0000
+++ generate-all	2011-02-09 12:26:14 +0000
@@ -39,6 +39,12 @@
     trend_starts = {}
     burnup_chart_teams = []
 
+lock_path = opts.database + ".generate_lock"
+lock_f = open(lock_path, "wb")
+if report_tools.lock_file(lock_f) is None:
+    print "Another instance is already running"
+    sys.exit(0)
+
 db = report_tools.get_db(opts.database)
 
 # get milestones and teams
@@ -64,10 +70,12 @@
 
 for u in users:
     for m in milestones:
-        # entire cycle report for user
         target = u + '-' + m
         basename = os.path.join(usersubdir, target)
         report_tools.run_reports(my_path, opts.database, basename, opts.config, milestone=m, user=u, trend_starts=trend_starts, burnup=u in burnup_chart_teams)
+    # entire cycle report for user
+    basename = os.path.join(usersubdir, u)
+    report_tools.run_reports(my_path, opts.database, basename, opts.config, milestone=None, user=u, trend_starts=trend_starts, burnup=u in burnup_chart_teams)
 
 # per team/milestone reports
 for t in teams:
@@ -103,3 +111,4 @@
     src.close()
     dest.close()
 
+os.unlink(lock_path)

=== modified file 'html-report'
--- html-report	2011-01-11 15:09:44 +0000
+++ html-report	2011-02-09 12:26:14 +0000
@@ -8,116 +8,87 @@
 from report_tools import escape_url, escape_html
 import report_tools
 
-def format_priority(priority):
-    prio_colors = {
-            'Undefined': 'gray',
-            'Low': 'blue',
-            'Medium': '#f60',
-            'High': 'red',
-            'Essential': 'red'
-    }
-
-    p = escape_html(priority or '')
-    col = prio_colors.get(p)
-    if col:
-        return '<span style="color: %s">%s</span>' % (col, p)
-    return p
+
+class WorkitemTarget(object):
+
+    def __init__(self, todo=0, blocked=0, inprogress=0, postponed=0,
+                 done=0):
+        self.todo = todo
+        self.blocked = blocked
+        self.inprogress = inprogress
+        self.postponed = postponed
+        self.done = done
+
+    @property
+    def percent_complete(self):
+        completed = self.postponed + self.done
+        total = completed + self.todo + self.blocked + self.inprogress
+        return int((float(completed)/total)*100 + 0.5)
+
+
+class Blueprint(WorkitemTarget):
+
+    def __init__(self, name, url, complexity=0,
+                 todo=0, blocked=0, inprogress=0, postponed=0,
+                 done=0, implementation=None, priority=None,
+                 status=None):
+        super(Blueprint, self).__init__(
+            todo=todo, blocked=blocked, inprogress=inprogress,
+            postponed=postponed, done=done)
+        self.name = name
+        self.url = url
+        self.complexity = complexity
+        self.implementation = implementation
+        self.priority = priority
+        self.status = status
+
+class Assignee(WorkitemTarget):
+
+    def __init__(self, name, url, complexity=0, todo_wis=[],
+                 blocked_wis=[], inprogress_wis=[], postponed_wis=[],
+                 done_wis=[]):
+        super(Assignee, self).__init__(
+            todo=len(todo_wis), blocked=len(blocked_wis),
+            inprogress=len(inprogress_wis), postponed=len(postponed_wis),
+            done=len(done_wis))
+        self.name = name
+        self.url = url
+        self.complexity = complexity
+        self.todo_wis = todo_wis
+        self.blocked_wis = blocked_wis
+        self.inprogress_wis = inprogress_wis
+        self.postponed_wis = postponed_wis
+        self.done_wis = done_wis
+
 
 def spec_completion(db, team, milestone):
     data = report_tools.blueprint_completion(db, team, milestone)
 
-    # avoid showing empty columns
-    has_priority = any([i['status'] for i in data.itervalues()])
-    has_complexity = any([i['complexity'] > 0 for i in data.itervalues()])
-
-    print '''<h2>Status by specification</h2>
-<p>(Click header to sort)</p>
-<table id="byspecification">
- <thead>
-  <tr><th>Specification</th><th>%s</th> <th>todo</th><th>blocked</th><th>inprogress</th><th>postponed</th><th>done</th> <th>Completion</th> <th>%s</th> <th>Status</th></tr>
- </thead>
-''' % (has_complexity and 'Complexity' or '', 
-       has_priority and 'Priority' or '')
-
-    completion = []
-    for (bp, i) in data.iteritems():
-        completion.append((bp,
-            int(float(i['postponed']+i['done'])/(i['todo']+i['blocked']+i['done']+i['postponed']+i['inprogress'])*100 + 0.5)))
-
-    completion.sort(key=lambda k: k[1]*100+report_tools.priority_value(data[k[0]]['priority']), reverse=True)
-
-    for (bp, percent) in completion:
-        print '  <tr><td>%s</td> <td>%s</td><td>%i</td><td>%i</td><td>%i</td><td>%i</td><td>%i</td> <td>%i%%<br/>' \
-            '<span style="font-size: 70%%">%s</span></td> <td>%s</td> <td>%s</td></tr>' % (
-            '<a href="%s">%s</a>' % (data[bp]['url'], escape_html(bp)),
-                data[bp]['complexity'] or '',
-                data[bp]['todo'], data[bp]['blocked'], data[bp]['inprogress'], data[bp]['postponed'], data[bp]['done'],
-                percent,
-                data[bp]['implementation'],
-                format_priority(data[bp]['priority']),
-                data[bp]['status'])
-
-    print '</table>'
-
-def by_assignee(db, team, milestone):
-    data = report_tools.assignee_completion(db, team, milestone)
-
-    # avoid showing empty columns
-    has_complexity = any([i['complexity'] > 0 for i in data.itervalues()])
-
-    print '''
-<h2>Status by assignee</h2>
-<p>(Click header to sort)</p>
-<table id="byassignee">
- <thead>
-  <tr><th>Assignee</th> <th>%s</th><th>todo</th><th>blocked</th><th>inprogress</th><th>postponed</th><th>done</th>  <th>Completion</th></tr>
- </thead>
-''' % (has_complexity and 'Complexity' or '', )
-
-    completion = []
-    for a, i in data.iteritems():
-        (todo, blocked, inprogress, postponed, done) = (len(i['todo']), len(i['blocked']), len(i['inprogress']), len(i['postponed']), len(i['done']))
-        completion.append((a,
-            int(float(postponed+done) / (todo+blocked+inprogress+done+postponed)*100 + 0.5)))
-
-    completion.sort(key=lambda k: k[0], reverse=False)
-
-    for (a, percent) in completion:
-        a_html = escape_url(a or team or 'nobody')
-        # no user pages exist w/o milestones, and its silly to link back to ourselves, so fall back to launchpad specs page
-        if not milestone or isinstance(team, report_tools.user_string):
-            url = '%s/~%s/+specs?role=assignee' % (report_tools.blueprints_base_url, a_html)
-        else:
-            target = a_html + '-' + milestone
-            url = 'u/%s.html' % target
-        print '  <tr><td><a href="%s">%s</a></td> <td>%s</td><td>%i</td><td>%i</td><td>%i</td><td>%i</td><td>%i</td> <td>%i%%</td></tr>' % (
-                url, a_html, data[a]['complexity'] or '', len(data[a]['todo']), len(data[a]['blocked']), len(data[a]['inprogress']), len(data[a]['postponed']),
-                len(data[a]['done']), percent)
-    print '</table>'
-
-def work_item_details(db, team, milestone):
-    data = report_tools.assignee_completion(db, team, milestone)
-
-    import sys
-
-    # avoid showing empty columns
-    has_priority = False
-    for i in data.itervalues():
-        for status in report_tools.valid_states:
-            for wi in i[status]:
-                if wi[2]:
-                    has_priority = True
-                    break
-        if has_priority:
-            break
-
-    print '''
-<h2>Work item details</h2>
-<table id="byworkitem">
- <thead>
-  <tr><th>Assignee</th> <th>Status</th> <th>Blueprint</th> <th>%s</th> <th>Work item</th></tr>
- </thead>
-''' % (has_priority and 'Priority' or '')
+    blueprints = []
+    for bp, i in data.items():
+        blueprints.append(
+            Blueprint(bp, i['url'], complexity=i['complexity'],
+                      todo=i['todo'], blocked=i['blocked'],
+                      inprogress=i['inprogress'],
+                      postponed=i['postponed'], done=i['done'],
+                      implementation=i['implementation'],
+                      priority=i['priority'], status=i['status']))
+
+    # avoid showing empty columns
+    specs_have_status = any([bp.status for bp in blueprints])
+    specs_have_complexity = any([bp.complexity > 0 for bp in blueprints])
+
+    blueprints.sort(
+        key=lambda bp: bp.percent_complete*100+report_tools.priority_value(bp.priority),
+        reverse=True)
+
+    return dict(
+        specs_have_status=specs_have_status,
+        specs_have_complexity=specs_have_complexity,
+        blueprints=blueprints)
+
+def get_assignee_completion(db, team, milestone):
+    data = report_tools.assignee_completion(db, team, milestone)
 
     def _sort(a, b):
         # Takes [blueprint, workitem, priority, url] in a and b.
@@ -128,38 +99,49 @@
         # then by blueprint
         return cmp(a[0], b[0])
 
-    for a, i in sorted(data.iteritems(), key=lambda k: k[0]):
-        todo_len = len(i['todo'])
-        blocked_len = len(i['blocked'])
-        postponed_len = len(i['postponed'])
-        done_len = len(i['done'])
-        inprogress_len = len(i['inprogress'])
-        a_html = escape_url(a or team or 'nobody')
-        a_url = '%s/~%s/+specs?role=assignee' % (report_tools.blueprints_base_url, a_html)
-        rows = {'todo': '<td rowspan="%s">todo</td>' % todo_len,
-                'blocked': '<td rowspan="%s">blocked</td>' % blocked_len,
-                'postponed': '<td rowspan="%s">postponed</td>' % postponed_len,
-                'done': '<td rowspan="%s">done</td>' % done_len,
-                'inprogress': '<td rowspan="%s">inprogress</td>' % inprogress_len}
-        printed_assignee = False
+    assignees = []
+    for name, info in data.items():
+        name = name or team or 'nobody'
+        name_html = escape_url(name or team or 'nobody')
+        if not milestone or isinstance(team, report_tools.user_string):
+            url = '%s/~%s/+specs?role=assignee' % (report_tools.blueprints_base_url, name_html)
+        else:
+            target = name_html + '-' + milestone
+            url = 'u/%s.html' % target
+        assignees.append(
+            Assignee(name, url, complexity=info['complexity'],
+                     todo_wis=sorted(info['todo'], cmp=_sort),
+                     blocked_wis=info['blocked'],
+                     inprogress_wis=info['inprogress'],
+                     postponed_wis=info['postponed'],
+                     done_wis=info['done']))
+    return assignees
+
+
+def by_assignee(db, team, milestone):
+    assignees = get_assignee_completion(db, team, milestone)
+    assignees.sort(key=lambda a: a.name, reverse=False)
+
+    # avoid showing empty columns
+    assignees_have_complexity = any([a.complexity > 0 for a in assignees])
+
+    # avoid showing empty columns
+    workitems_have_priority = False
+    for assignee in assignees:
         for status in report_tools.valid_states:
-            printed_status = False
-            i[status].sort(cmp=_sort)
-            for (bp, item, priority, url) in i[status]:
-                print '  <tr>',
-                if not printed_assignee:
-                    print '<td rowspan="%s"><a href="%s" name="%s">%s</a></td> ' % (
-                        todo_len+blocked_len+postponed_len+done_len+inprogress_len, a_url, escape_url(a or team or 'nobody'),
-                        a_html)
-                    printed_assignee = True
-                if not printed_status:
-                    print rows[status]
-                    printed_status = True
-                print '<td>%s</td> <td>%s</td> <td>%s</td></tr>' % (
-                    '<a href="%s">%s</a>' % (url, escape_html(bp)),
-                    format_priority(priority),
-                    item)
-    print '</table>'
+            for wi in getattr(assignee, status+"_wis", []):
+                if wi[2]:
+                    workitems_have_priority = True
+                    break
+        if workitems_have_priority:
+            break
+
+    return dict(
+        assignees_have_complexity=assignees_have_complexity,
+        assignees=assignees,
+        workitems_have_priority=workitems_have_priority,
+        valid_states=report_tools.valid_states)
+
 
 def html(db, team, milestone, chart_url, title=None):
     '''Print work item status as HTML.'''
@@ -173,118 +155,30 @@
     if milestone:
         title += ' (%s)' % milestone
 
-    print '''<html>
-<head>
-  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-  <title>Work item status: %s</title>
-  <style type="text/css">
-    body { background: #F0F0E0; color: black; }
-    a { text-decoration: none; }
-    table { border-collapse: collapse; border-style: solid none;
-            border-width: 3px; margin-bottom: 3ex; empty-cells: show; }
-    table th { text-align: left; border-style: none none solid none;
-               border-width: 3px; padding-right: 10px; }
-    table td { text-align: left; border-style: none none dotted none;
-               border-width: 1px; padding-right: 10px; }
-    table#byworkitem td { vertical-align: top; }
-
-    a { color: blue; }
-  </style>
-
-
-  <script type="text/javascript" src="jquery-1.4.min.js"></script>
-  <script type="text/javascript" src="jquery.tablesorter.min.js"></script>
-
-  <script type="text/javascript">
-    // documentation at: http://tablesorter.com/
-
-    $(document).ready( function() {
-        // add parser through the tablesorter addParser method
-        $.tablesorter.addParser({
-            // set a unique id
-            id: 'priority',
-            is: function(s) {
-                // return false so this parser is not auto detected
-                return false;
-            },
-            format: function(s) {
-                // format your data for normalization
-                return s.toLowerCase()
-                    .replace(/essential/,5)
-                    .replace(/high/,4)
-                    .replace(/medium/,3)
-                    .replace(/low/,2)
-                    .replace(/undefined/,1)
-                    .replace(/not/,0);
-
-            },
-            // set type, either numeric or text
-            type: 'numeric'
-        });
-
-        $.tablesorter.addParser({
-            // set a unique id
-            id: 'completion',
-            is: function(s) {
-                // return false so this parser is not auto detected
-                return false;
-            },
-            format: function(s) {
-                // format your data for normalization
-                return s.replace(/^(\d+).*/,'\1');
-
-            },
-            // set type, either numeric or text
-            type: 'numeric'
-        });
-
-        // order they appear on the page
-        $("#byspecification").tablesorter({
-            // sort on priority, spec ascending
-            sortList: [[5,1],[0,0]],
-            headers: {
-                4: { sorter:'completion' },
-                5: { sorter:'priority' }
-            }
-        });
-
-        $("#byassignee").tablesorter({
-            // sort on name, ascending
-            sortList: [[0,0]]
-        });
-
-        // with the 'todo/done' and assignee not appearing on every
-        // row, this does not lend itself well to sorting
-        //$("#byworkitem").tablesorter();
-
-    });
-  </script>
-</head>
-
-<body>
-
-<h1>Burndown for %s</h1>
-
-<p><object height="600" width="1000" data="%s" type="image/svg+xml">Burndown SVG</object></p>
-''' % (title, title, chart_url)
-
-    spec_completion(db, team, milestone)
-    by_assignee(db, team, milestone)
-    work_item_details(db, team, milestone)
-    time_stats(db, team, milestone)
-
-    print '</body></html>'
+    data = dict(title=title, chart_url=chart_url)
+    data.update(spec_completion(db, team, milestone))
+    data.update(by_assignee(db, team, milestone))
+    data.update(time_stats(db, team, milestone))
+    print report_tools.fill_template("burndown.html", data)
+
+
+class WorkitemsOnDate(object):
+
+    def __init__(self, date, done, delta_done, todo, delta_todo):
+        self.date = date
+        self.done = done
+        self.delta_done = delta_done
+        self.todo = todo
+        self.delta_todo = delta_todo
+
 
 def time_stats(db, team, milestone):
     data = report_tools.workitems_over_time(db, team, milestone)
 
-    print '''
-<h2>Development over time</h2>
-<table id="time_stats">
-  <tr><th>Date</th><th>done</th><th>todo</th></tr>
-'''
     prev_done = None
     prev_todo = None
+
+    workitems_by_day = []
     for (date, states) in sorted(data.iteritems(), key=lambda k: k[0]):
         todo = states.get('todo', 0) + states.get('inprogress', 0)
         done = states.get('done', 0) + states.get('postponed', 0)
@@ -306,10 +200,9 @@
             delta_todo = ''
         prev_todo = todo
         prev_done = done
-        print ' <tr><td>%s</td><td>%i%s</td><td>%i%s</td></tr>' % (
-                date, done, delta_done, todo, delta_todo)
-
-print '</table>'
+        workitems_by_day.append(WorkitemsOnDate(date, done, delta_done,
+            todo, delta_todo))
+    return dict(workitems_by_day=workitems_by_day)
 
 #
 # main

=== modified file 'report_tools.py'
--- report_tools.py	2010-12-03 12:41:46 +0000
+++ report_tools.py	2011-02-09 12:26:14 +0000
@@ -6,6 +6,8 @@
 import sqlite3 as dbapi2
 from subprocess import Popen
 from cgi import escape
+import errno
+import fcntl
 
 valid_states = ['inprogress','blocked','todo','done', 'postponed']
 blueprints_base_url = 'https://blueprints.launchpad.net'
@@ -547,6 +549,21 @@
 
     return trans.get(name, 0)
 
+def lock_file(f):
+    try:
+        fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        return f
+    except IOError, e:
+        if e.errno in (errno.EACCES, errno.EAGAIN):
+            return None
+        raise
+
+def fill_template(name, data):
+    from mako.lookup import TemplateLookup
+    lookup = TemplateLookup(directories=['templates'])
+    template = lookup.get_template(name)
+    return template.render_unicode(**data)
+
 if __name__ == '__main__':
     # some test stuff
     db = get_db(sys.argv[1])

=== added directory 'templates'
=== added file 'templates/burndown.html'
--- templates/burndown.html	1970-01-01 00:00:00 +0000
+++ templates/burndown.html	2011-02-09 12:26:14 +0000
@@ -0,0 +1,241 @@
+<html>
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+  <title>Work item status: %{title}</title>
+  <style type="text/css">
+    body { background: #F0F0E0; color: black; }
+    a { text-decoration: none; }
+    table { border-collapse: collapse; border-style: solid none;
+            border-width: 3px; margin-bottom: 3ex; empty-cells: show; }
+    table th { text-align: left; border-style: none none solid none;
+               border-width: 3px; padding-right: 10px; }
+    table td { text-align: left; border-style: none none dotted none;
+               border-width: 1px; padding-right: 10px; }
+    table#byworkitem td { vertical-align: top; }
+
+    a { color: blue; }
+
+    td.priority_Undefined { color: gray; }
+    td.priority_Low { color: blue; }
+    td.priority_Medium { color: #f60; }
+    td.priority_High { color: red; }
+    td.priority_Essential { color: red; }
+    td span.implementation_status { font-size: 70%; }
+  </style>
+
+
+  <script type="text/javascript" src="jquery-1.4.min.js"></script>
+  <script type="text/javascript" src="jquery.tablesorter.min.js"></script>
+
+  <script type="text/javascript">
+    // documentation at: http://tablesorter.com/
+
+    $(document).ready( function() {
+        // add parser through the tablesorter addParser method
+        $.tablesorter.addParser({
+            // set a unique id
+            id: 'priority',
+            is: function(s) {
+                // return false so this parser is not auto detected
+                return false;
+            },
+            format: function(s) {
+                // format your data for normalization
+                return s.toLowerCase()
+                    .replace(/essential/,5)
+                    .replace(/high/,4)
+                    .replace(/medium/,3)
+                    .replace(/low/,2)
+                    .replace(/undefined/,1)
+                    .replace(/not/,0);
+
+            },
+            // set type, either numeric or text
+            type: 'numeric'
+        });
+
+        $.tablesorter.addParser({
+            // set a unique id
+            id: 'completion',
+            is: function(s) {
+                // return false so this parser is not auto detected
+                return false;
+            },
+            format: function(s) {
+                // format your data for normalization
+                return s.replace(/^(\d+).*/,'\1');
+
+            },
+            // set type, either numeric or text
+            type: 'numeric'
+        });
+
+        // order they appear on the page
+        $("#byspecification").tablesorter({
+            // sort on priority, spec ascending
+            sortList: [[5,1],[0,0]],
+            headers: {
+                4: { sorter:'completion' },
+                5: { sorter:'priority' }
+            }
+        });
+
+        $("#byassignee").tablesorter({
+            // sort on name, ascending
+            sortList: [[0,0]]
+        });
+
+        // with the 'todo/done' and assignee not appearing on every
+        // row, this does not lend itself well to sorting
+        //$("#byworkitem").tablesorter();
+
+    });
+  </script>
+</head>
+
+<body>
+
+<h1>Burndown for ${title}</h1>
+
+<p><object height="600" width="1000" data="${chart_url}" type="image/svg+xml">Burndown SVG</object></p>
+
+<h2>Status by specification</h2>
+<p>(Click header to sort)</p>
+<table id="byspecification">
+ <thead>
+	 <tr>
+		 <th>Specification</th>
+% if specs_have_complexity:
+		 <th>Complexity</th>
+% endif
+		 <th>todo</th>
+		 <th>blocked</th>
+		 <th>inprogress</th>
+		 <th>postponed</th>
+		 <th>done</th>
+		 <th>Completion</th>
+		 <th>Priority</th>
+% if specs_have_status:
+		 <th>Status</th>
+% endif
+	 </tr>
+ </thead>
+% for bp in blueprints:
+ <tr>
+	 <td><a href="${bp.url}">${bp.name}</a></td>
+% if has_complexity:
+	 <td>${bp.complexity}</td>
+% endif
+	 <td>${bp.todo}</td>
+	 <td>${bp.blocked}</td>
+	 <td>${bp.inprogress}</td>
+	 <td>${bp.postponed}</td>
+	 <td>${bp.done}</td>
+	 <td>${bp.percent_complete}%<br />
+	     <span class="implementation_status">${bp.implementation}</span></td>
+	 <td class="priority_${bp.priority}">${bp.priority}</td>
+	 <td>${bp.status}</td>
+ </tr>
+% endfor
+</table>
+
+<h2>Status by assignee</h2>
+<p>(Click header to sort)</p>
+<table id="byassignee">
+<thead>
+	<tr>
+		<th>Assignee</th>
+% if assignees_have_complexity:
+		<th>Complexity</th>
+% endif
+		<th>todo</th>
+		<th>blocked</th>
+		<th>inprogress</th>
+		<th>postponed</th>
+		<th>done</th>
+		<th>Completion</th>
+	</tr>
+</thead>
+% for a in assignees:
+	<tr>
+		<td><a href="${a.url}">${a.name}</a></td>
+% if assignees_have_complexity:
+		<td>${a.complexity}</td>
+% endif
+		<td>${a.todo}</td>
+		<td>${a.blocked}</td>
+		<td>${a.inprogress}</td>
+		<td>${a.postponed}</td>
+		<td>${a.done}</td>
+		<td>${a.percent_complete}%</td>
+	</tr>
+% endfor
+</table>
+
+<h2>Work item details</h2>
+<table id="byworkitem">
+<thead>
+	<tr>
+		<th>Assignee</th>
+		<th>Status</th>
+		<th>Blueprint</th>
+% if workitems_have_priority:
+		<th>Priority</th>
+% endif
+		<th>Work item</th>
+	</tr>
+</thead>
+
+% for a in assignees:
+<%
+    printed_assignee = False
+%>
+% for status in valid_states:
+<%
+    printed_status = False
+%>
+% for bp, item, priority, url in getattr(a, status+"_wis"):
+	<tr>
+% if not printed_assignee:
+		<td rowspan="${a.todo+a.blocked+a.postponed+a.done+a.inprogress}"><a href="${a.url}" name="${a.name}">${a.name}</a></td>
+		<%
+		printed_assignee = True
+		%>
+% endif
+% if not printed_status:
+		<td rowspan="${getattr(a, status)}">${status}</td>
+		<%
+		printed_status = True
+		%>
+% endif
+		<td><a href="${url}">${bp}</a></td>
+% if workitems_have_priority:
+		<td class="priority_${priority}">${priority}</td>
+% endif
+		<td>${item}</td>
+	</tr>
+% endfor
+% endfor
+% endfor
+
+</table>
+
+<h2>Development over time</h2>
+<table id="time_stats">
+  <thead>
+    <tr>
+        <th>Date</th>
+        <th>done</th>
+        <th>todo</th>
+    </tr>
+  </thead>
+% for count_on_day in workitems_by_day:
+    <tr>
+        <td>${count_on_day.date}</td>
+        <td>${count_on_day.done}${count_on_day.delta_done}</td>
+        <td>${count_on_day.todo}${count_on_day.delta_todo}</td>
+    </tr>
+% endfor
+</table>
+</body>
+</html>

=== added file 'templates/test.html'
--- templates/test.html	1970-01-01 00:00:00 +0000
+++ templates/test.html	2011-02-09 12:26:14 +0000
@@ -0,0 +1,1 @@
+Testing ${data}

=== added file 'tests.py'
--- tests.py	1970-01-01 00:00:00 +0000
+++ tests.py	2011-02-09 12:26:14 +0000
@@ -0,0 +1,16 @@
+from testtools import TestCase
+import report_tools
+
+
+class FillTemplateTests(TestCase):
+
+    def test_finds_template(self):
+        output = report_tools.fill_template("test.html", dict(data="foo"))
+        self.assertEqual("Testing foo\n", output)
+
+
+def test_suite():
+    from unittest import TestLoader
+    loader = TestLoader()
+    suite = loader.loadTestsFromName(__name__)
+    return suite