Merge lp:~milo/launchpad-work-items-tracker/bug996948 into lp:launchpad-work-items-tracker

[Milo Casagrande]

Merge proposal in order to fix the integrity error we receive from status.l.o.

Changes done:
- catch the integrity error and print warning with name of duplicated Blueprint
- skip the Blueprint and continue, otherwise script will fail later in the code due to variable not being set

[lp:~milo/launchpad-work-items-tracker/bug996948]

344. By Milo Casagrande on 2012-10-09

Used error logger, added URL to Blueprint.

Unmerged revisions

344. By Milo Casagrande on 2012-10-09

Used error logger, added URL to Blueprint.

343. By Milo Casagrande on 2012-10-09

Catch integrity error, and print warning.

342. By James Tunnicliffe on 2012-08-08

Removed template driven excaping from some pre-defined HTML so it renders correctly

341. By James Tunnicliffe on 2012-08-08

Fixed errors parsing meta values containing a colon

340. By Milo Casagrande on 2012-08-03

Merged lp:~lool/launchpad-work-items-tracker/fix-line-wrap-breakage.

339. By Milo Casagrande on 2012-08-03

Merge lp:~lool/launchpad-work-items-tracker/jira-support.

338. By Данило Шеган on 2012-07-17

Merge HTML escaping fix from trunk r301 (by Martin Pitt).

337. By Данило Шеган on 2012-07-09

Fix problem with graph generation for milestones in the far future. Patch by Loic.

336. By Milo Casagrande on 2012-07-02

Merge lp:~milo/launchpad-work-items-tracker/bug1017878.

335. By James Tunnicliffe on 2012-05-16

Merge in migration tools

=== modified file 'all-projects'
--- all-projects	2012-08-27 15:32:43 +0000
+++ all-projects	2012-10-09 09:20:30 +0000
@@ -14,9 +14,27 @@
 import sys
 import datetime
 
+import report_tools
+
 
 def collect(source_dir, db_file, config_file, extra_args):
-    args = [os.path.join(source_dir, "collect")]
+    return run_collect_script(source_dir, db_file, config_file, extra_args,
+                              "collect")
+
+
+def collect_jira(source_dir, db_file, config_file, extra_args):
+    return run_collect_script(source_dir, db_file, config_file, extra_args,
+                              "collect_jira")
+
+
+def collect_roadmap(source_dir, db_file, config_file, extra_args):
+    return run_collect_script(source_dir, db_file, config_file, extra_args,
+                              "collect_roadmap")
+
+
+def run_collect_script(source_dir, db_file, config_file, extra_args,
+                       collect_script):
+    args = [os.path.join(source_dir, collect_script)]
     args.extend(["-d", db_file])
     args.extend(["-c", config_file])
     args += extra_args
@@ -56,9 +74,14 @@
 
 
 def main():
-    parser = optparse.OptionParser(usage="%prog <database dir> <www root dir> [www root url]")
+    parser = optparse.OptionParser(
+        usage="%prog <database dir> <www root dir> [www root url]")
     parser.add_option("--config-dir", dest="config_dir", default="config")
+    parser.add_option("--roadmap-config-file", dest="roadmap_config_file",
+                      default="roadmap-config")
     parser.add_option("--debug", dest="debug", action="store_true")
+    parser.add_option("--kanban-token-file", dest="kanban_token_file")
+    parser.add_option("--papyrs-token-file", dest="papyrs_token_file")
     opts, args = parser.parse_args()
 
     if os.environ.get("DEBUG", None) is not None:
@@ -88,6 +111,11 @@
         os.path.join(source_dir, opts.config_dir, "*%s" % valid_config_suffix))
 
     for config_file in filenames:
+        # read roadmap config to find where to get cards from
+        cfg = report_tools.load_config(config_file)
+        # default to kanbantool
+        cards_source = cfg.get('cards_source', 'kanban')
+
         project_name = os.path.basename(config_file)[:-len(valid_config_suffix)]
         project_output_dir = os.path.join(output_dir, project_name)
         db_file = os.path.join(db_dir, "%s.db" % project_name)
@@ -111,6 +139,40 @@
         if not collect(source_dir, db_file, config_file, extra_collect_args):
             sys.stderr.write("collect failed for %s" % project_name)
             continue
+
+        if cards_source == 'jira':
+            extra_collect_jira_args = []
+            if not collect_jira(source_dir, db_file, opts.roadmap_config_file,
+                                extra_collect_jira_args):
+                sys.stderr.write("collect_jira failed for %s" % project_name)
+                continue
+        elif cards_source == 'kanban':
+            extra_collect_roadmap_args = []
+            extra_collect_roadmap_args.extend(["--board", '10721'])
+            if opts.kanban_token_file is not None:
+                with open(opts.kanban_token_file) as token_file:
+                    token = token_file.read()
+                    extra_collect_roadmap_args.extend(["--kanbantoken", token])
+            else:
+                sys.stderr.write("No Kanbantool API token given to "
+                                 "collect_roadmap for %s" % project_name)
+            if opts.papyrs_token_file is not None:
+                with open(opts.papyrs_token_file) as token_file:
+                    token = token_file.read()
+                    extra_collect_roadmap_args.extend(["--papyrstoken", token])
+            else:
+                sys.stderr.write("No Papyrs API token given to "
+                                 "collect_roadmap for %s" % project_name)
+
+            if not collect_roadmap(source_dir, db_file,
+                                   opts.roadmap_config_file,
+                                   extra_collect_roadmap_args):
+                sys.stderr.write("collect_roadmap failed for %s" %
+                                 project_name)
+        else:
+            sys.stderr.write("Unknown cards source %s" % cards_source)
+            continue
+
         publish_new_db(project_name, project_output_dir, db_file)
         generate_reports(project_output_dir, config_file, db_file,
             source_dir, extra_generate_args, debug=opts.debug)

=== modified file 'burndown-chart'
--- burndown-chart	2012-08-24 05:29:13 +0000
+++ burndown-chart	2012-10-09 09:20:30 +0000
@@ -312,6 +312,11 @@
         opts.team or 'all', opts.group or 'none', milestone_collection and milestone_collection.display_name or 'none')
     sys.exit(0)
 
+if date_to_ordinal(end_date) - date_to_ordinal(start_date) > 366:
+    print 'WARNING: date range %s - %s is over a year, not generating a chart' % (
+        start_date, end_date)
+    sys.exit(0)
+
 # title
 if opts.team:
     title = '/20' + opts.team

=== modified file 'collect'
--- collect	2012-07-11 22:36:56 +0000
+++ collect	2012-10-09 09:20:30 +0000
@@ -6,13 +6,27 @@
 # Copyright (C) 2010, 2011 Canonical Ltd.
 # License: GPL-3
 
-import urllib, re, sys, optparse, smtplib, pwd, os, urlparse
 import logging
+import optparse
+import os
+import pwd
+import re
+import smtplib
+import sys
+import urllib
+import urlparse
+import sqlite3
+
 from email.mime.text import MIMEText
 
 from launchpadlib.launchpad import Launchpad, EDGE_SERVICE_ROOT
 
-from lpworkitems.collect import CollectorStore, PersonCache, WorkitemParser, bug_wi_states
+from lpworkitems.collect import (
+    CollectorStore,
+    PersonCache,
+    WorkitemParser,
+    bug_wi_states,
+    )
 from lpworkitems.database import get_store
 from lpworkitems.error_collector import (
     BlueprintURLError,
@@ -26,6 +40,7 @@
     TeamParticipation,
     Workitem,
     )
+from utils import unicode_or_None
 import report_tools
 
 
@@ -58,20 +73,16 @@
     """Get a link to the Launchpad API object on the website."""
     api_link = item.self_link
     parts = urlparse.urlparse(api_link)
-    link = parts.scheme + "://" + parts.netloc.replace("api.", "") + "/" + parts.path.split("/", 2)[2]
+    link = parts.scheme + "://" + parts.netloc.replace("api.", "") + \
+           "/" + parts.path.split("/", 2)[2]
     return link.decode("utf-8")
 
 
-def unicode_or_None(attr):
-    if attr is None:
-        return attr
-    if isinstance(attr, unicode):
-        return attr
-    return attr.decode("utf-8")
-
-
 import simplejson
+
 _orig_loads = simplejson.loads
+
+
 def loads(something):
     return _orig_loads(unicode_or_None(something))
 simplejson.loads = loads
@@ -96,7 +107,10 @@
     '''
     model_bp = Blueprint.from_launchpad(bp)
     if model_bp.milestone_name not in collector.valid_milestone_names():
-        data_error(web_link(bp), 'milestone "%s" is unknown/invalid' % model_bp.milestone_name, True)
+        data_error(
+            web_link(bp),
+            'milestone "%s" is unknown/invalid' % model_bp.milestone_name,
+            True)
     model_bp = collector.store_blueprint(model_bp)
     if model_bp:
         dbg('lp_import_blueprint: added blueprint: %s' % bp.name)
@@ -116,7 +130,10 @@
     model_group = BlueprintGroup.from_launchpad(bp)
     model_group.area = area
     if model_group.milestone_name not in collector.valid_milestone_names():
-        data_error(web_link(bp), 'milestone "%s" is unknown/invalid' % model_group.milestone, True)
+        data_error(
+            web_link(bp),
+            'milestone "%s" is unknown/invalid' % model_group.milestone,
+            True)
 
     model_group = collector.store_blueprint_group(model_group)
     if model_group is None:
@@ -126,9 +143,9 @@
     add_dependencies(collector, model_group.name, deps)
     return model_group
 
+
 def parse_meta_item(collector, line, bp_name):
     '''Parse a meta information line from a blueprint
-
     '''
 
     line = line.strip()
@@ -136,18 +153,19 @@
         return
 
     try:
-        (key, value) = line.rsplit(':', 1)
-        key   = key.strip()
+        (key, value) = line.split(':', 1)
+        key = key.strip()
         value = value.strip()
     except ValueError:
         dbg("\tMeta line '%s' can not be parsed" % line)
         return
 
-    dbg( "\t\tMeta for %s: key='%s' value='%s'" % (bp_name, key, value) )
+    dbg("\t\tMeta for %s: key='%s' value='%s'" % (bp_name, key, value))
     collector.store_meta(key, value, bp_name)
 
 
-def parse_complexity_item(collector, line, bp_name, bp_url, def_milestone, def_assignee):
+def parse_complexity_item(collector, line, bp_name, bp_url, def_milestone,
+                          def_assignee):
     line = line.strip()
     # remove special characters people tend to type
     line = re.sub('[^\w -.]', '', line)
@@ -156,9 +174,9 @@
 
     dbg("\tParsing complexity line '%s'" % line)
 
-    num         = None
-    milestone   = None
-    assignee    = None
+    num = None
+    milestone = None
+    assignee = None
 
     try:
         complexity_list = line.split()
@@ -176,7 +194,9 @@
         dbg('\tComplexity: %s MS: %s Who: %s' % (num, milestone, assignee))
         collector.store_complexity(assignee, num, milestone, bp_name)
     except ValueError:
-        data_error(bp_url, "\tComplexity line '%s' could not be parsed %s" % (line, ValueError))
+        data_error(bp_url,
+            "\tComplexity line '%s' could not be parsed %s" %
+                (line, ValueError))
 
 
 def milestone_extract(text, valid_milestones):
@@ -188,7 +208,9 @@
             return word
     return None
 
-def lp_import_blueprint_workitems(collector, bp, distro_release, people_cache=None, projects=None):
+
+def lp_import_blueprint_workitems(collector, bp, distro_release,
+                                  people_cache=None, projects=None):
     '''Collect work items from a Launchpad blueprint.
 
     This includes work items from the whiteboard as well as linked bugs.
@@ -202,17 +224,26 @@
     in_complexity_block = False
     work_items = []
 
-    model_bp = collector.store.find(
-        Blueprint, Blueprint.name == bp.name).one()
-    assert model_bp is not None, "Asked to process workitems of %s when it is not in the db" % bp.name
-
-    dbg('lp_import_blueprint_workitems(): processing %s (spec milestone: %s, spec assignee: %s, spec implementation: %s)' % (
+    try:
+        model_bp = collector.store.find(Blueprint,
+                                        Blueprint.name == bp.name).one()
+    except sqlite3.IntegrityError:
+        logger.warn('Duplicated Blueprint found: %s. It will not be '
+                    'considered.' % bp.name)
+        return
+
+    assert model_bp is not None, \
+        "Asked to process workitems of %s when it is not in the db" % bp.name
+
+    dbg('lp_import_blueprint_workitems(): processing %s (spec milestone: %s,' \
+        ' spec assignee: %s, spec implementation: %s)' % (
         bp.name, model_bp.milestone_name, model_bp.assignee_name,
         model_bp.implementation))
 
     valid_milestones = collector.valid_milestone_names()
     global error_collector
     parser = WorkitemParser(
+<<<<<<< TREE
         model_bp, model_bp.milestone_name, collector.lp, people_cache=people_cache,
         error_collector=error_collector)
 
@@ -228,19 +259,43 @@
     if workitems_text:
         for l in workitems_text.splitlines():
             if not in_workitems_block:
+=======
+        model_bp, model_bp.milestone_name, collector.lp,
+        people_cache=people_cache, error_collector=error_collector)
+
+    # Get work items from both the whiteboard and the new workitems_text
+    # property. Once the migration is completed and nobody's using the
+    # whiteboard for work items we can change this to pull work items only
+    # from bp.workitems_text.
+    workitems_text = bp.whiteboard
+    if workitems_text is None:
+        workitems_text = ''
+    if getattr(bp, 'workitems_text', '') != '':
+        workitems_text += "\n" + bp.workitems_text
+    if workitems_text:
+        for l in workitems_text.splitlines():
+            if not in_workitems_block:
+>>>>>>> MERGE-SOURCE
                 m = work_items_re.search(l)
                 if m:
                     in_workitems_block = True
-                    dbg('lp_import_blueprint_workitems(): starting work items block at ' + l)
+                    dbg('lp_import_blueprint_workitems():'
+                        ' starting work items block at ' + l)
                     milestone = milestone_extract(m.group(1), valid_milestones)
                     dbg('  ... setting milestone to ' + str(milestone))
+<<<<<<< TREE
                     parser.milestone_name = milestone or parser.blueprint.milestone_name
+=======
+                    parser.milestone_name = \
+                        milestone or parser.blueprint.milestone_name
+>>>>>>> MERGE-SOURCE
                 continue
 
             if in_workitems_block:
                 dbg("\tworkitem (raw): '%s'" % (l.strip()))
                 if not l.strip():
-                    dbg('lp_import_blueprint_workitems(): closing work items block with line: ' + l)
+                    dbg('lp_import_blueprint_workitems():'
+                        ' closing work items block with line: ' + l)
                     in_workitems_block = False
                     parser.milestone_name = parser.blueprint.milestone_name
                 workitem = parser.parse_blueprint_workitem(l)
@@ -308,7 +363,8 @@
                 member.name, team.name)
             if recursive or team.name in cfg.get('recursive_teams', []):
                 _import_teams_recurse(
-                        collector, cfg, member, top_level_team_names + [member.name],
+                        collector, cfg, member,
+                        top_level_team_names + [member.name],
                         people_cache=people_cache, recursive=True)
 
 
@@ -346,12 +402,14 @@
     blueprint.status = lp_project.summary or name
     collector.store_blueprint(blueprint)
 
-    for task in lp_project.searchTasks(status=bug_wi_states.keys(), **cfg['work_item_bugs']):
+    for task in lp_project.searchTasks(status=bug_wi_states.keys(),
+                                       **cfg['work_item_bugs']):
         id = task.self_link.split('/')[-1]
         title = task.title.split('"', 1)[1].rstrip('"')
         state = bug_wi_states[task.status]
         if state is None:
-            dbg('lp_import_bug_workitems: ignoring #%s: %s (status: %s)' % (id, title, task.status))
+            dbg('lp_import_bug_workitems: ignoring #%s: %s (status: %s)' % (
+                    id, title, task.status))
             continue
         dbg('lp_import_bug_workitems: #%s: %s (%s)' % (id, title, state))
 
@@ -384,14 +442,17 @@
         milestones.extend([ms for ms in project.all_milestones])
 
     if 'release' in cfg:
-        lp_project = collector.lp.distributions['ubuntu'].getSeries(name_or_version=cfg['release'])
+        lp_project = collector.lp.distributions['ubuntu'].getSeries(
+                         name_or_version=cfg['release'])
         projects.append((lp_project, None))
         add_milestones(lp_project)
     else:
-        assert 'project' in cfg, 'Configuration needs to specify project or release'
+        assert 'project' in cfg, \
+            'Configuration needs to specify project or release'
         lp_project = collector.lp.projects[cfg['project']]
         if 'project_series' in cfg:
-            lp_project_series = lp_project.getSeries(name=cfg['project_series'])
+            lp_project_series = lp_project.getSeries(
+                 name=cfg['project_series'])
             add_milestones(lp_project_series)
         else:
             lp_project_series = None
@@ -413,6 +474,10 @@
         if is_dict and extra_projects[extra_project_name] is not None:
             extra_project_series = extra_project.getSeries(
                 name=extra_projects[extra_project_name])
+            if extra_project_series is None:
+                raise AssertionError(
+                    "%s has no series named %s"
+                    % (extra_project_name, extra_projects[extra_project_name]))
             add_milestones(extra_project_series)
         else:
             extra_project_series = None
@@ -453,10 +518,12 @@
 
     for project, series in projects:
     # XXX: should this be valid_ or all_specifications?
-        project_spec_group_matcher = spec_group_matchers.get(project.name, None)
+        project_spec_group_matcher = spec_group_matchers.get(project.name,
+                                                             None)
         project_bps = project.valid_specifications
         for bp in project_bps:
-            if name_pattern is not None and re.search(name_pattern, bp.name) is None:
+            if name_pattern is not None and \
+                re.search(name_pattern, bp.name) is None:
                 continue
             if project_spec_group_matcher is not None:
                 match = re.search(project_spec_group_matcher, bp.name)
@@ -471,7 +538,8 @@
                 add_blueprint(bp)
         if series is not None:
             for bp in series.valid_specifications:
-                if name_pattern is not None and re.search(name_pattern, bp.name) is None:
+                if name_pattern is not None and \
+                    re.search(name_pattern, bp.name) is None:
                     continue
                 if project_spec_group_matcher is not None:
                     match = re.search(project_spec_group_matcher, bp.name)
@@ -492,7 +560,6 @@
                 deps[possible_dep] = possible_deps[possible_dep]
         if deps:
             lp_import_spec_group(collector, spec_group, area, deps)
-
     lp_import_bug_workitems(lp_project, collector, cfg)
 
 
@@ -540,7 +607,8 @@
             if in_section:
                 result.append([name, status, section])
             fields = line.strip().split(u'==')
-            assert not fields[0] # should be empty
+            # should be empty
+            assert not fields[0]
             name = fields[1].strip()
             section = []
             collect = 1
@@ -550,7 +618,8 @@
                 in_section = True
             collect = 0
             fields = line.strip().split(u'||')
-            assert not fields[0] # should be empty
+            # should be empty
+            assert not fields[0]
             assignee = default_assignee
             istatus = u'todo'
             milestone = None
@@ -562,11 +631,12 @@
             desc = fields[which].strip()
             if u'status' in field_off:
                 which = field_off[u'status']
-                status_search = [ fields[which] ]
+                status_search = [fields[which]]
             else:
                 status_search = fields[2:]
             for f in status_search:
-                if u'DONE' in f or u'POSTPONED' in f or u'TODO' in f or u'INPROGRESS' in f or u'BLOCKED' in f:
+                if u'DONE' in f or u'POSTPONED' in f or u'TODO' in f or \
+                    u'INPROGRESS' in f or u'BLOCKED' in f:
                     ff = f.split()
                     if len(ff) == 2:
                         assignee = ff[1]
@@ -615,14 +685,17 @@
     for url, default_assignee in cfg.get('moin_pages', {}).iteritems():
         url = unicode_or_None(url)
         default_assignee = unicode_or_None(default_assignee)
-        dbg('moin_import(): processing %s (default assignee: %s)' % (url, default_assignee))
-        for group, status, items in get_moin_workitems_group(url, default_assignee):
+        dbg('moin_import(): processing %s (default assignee: %s)' % (
+                url, default_assignee))
+        for group, status, items in get_moin_workitems_group(url,
+                                                             default_assignee):
             url_clean = url.replace('?action=raw', '')
             name = url_clean.split('://', 1)[1].split('/', 1)[1]
             if group:
                 name += u' ' + group
                 spec_url = u'%s#%s' % (url_clean, escape_url(group))
-                dbg('  got group %s: name="%s", url="%s"' % (group, name, spec_url))
+                dbg('  got group %s: name="%s", url="%s"' % (
+                        group, name, spec_url))
             else:
                 spec_url = url_clean
                 dbg('  no group: name="%s", url="%s"' % (name, spec_url))
@@ -661,7 +734,8 @@
     optparser.add_option('-c', '--config',
         help='Path to configuration file', dest='config', metavar='PATH')
     optparser.add_option('-p', '--pattern', metavar='REGEX',
-        help='Regex pattern for blueprint name (optional, mainly for testing)', dest='pattern')
+        help='Regex pattern for blueprint name (optional, mainly for testing)',
+        dest='pattern')
     optparser.add_option('--debug', action='store_true', default=False,
         help='Enable debugging output in parsing routines')
     optparser.add_option('--mail', action='store_true', default=False,
@@ -679,41 +753,54 @@
 
     return opts, args
 
+
 def send_error_mails(cfg):
     '''Send data_errors to contacts.
 
-    Data error contacts are defined in the configuration in the "error_contact"
-    map (which assigns a regexp over spec names to a list of email addresses).
-    If no match is found, the error goes to stderr.
+    Data error contacts are defined in the configuration in the
+    "project_notification_addresses" map (which assigns project names to a list
+    of email addresses). If no address list for a project is found, the error
+    goes to stderr.
     '''
     global error_collector
 
     # sort errors into address buckets
-    emails = {} # email address -> contents
+    # email address -> contents
+    emails = {}
 
     dbg('mailing %i data errors' % len(error_collector.errors))
     for error in error_collector.errors:
-        for pattern, addresses in cfg['error_contact'].iteritems():
-            if (error.get_blueprint_name() is not None
-                and re.search(pattern, error.get_blueprint_name())):
-                dbg('spec %s matches error_contact pattern "%s", mailing to %s' % (error.get_blueprint_name(),
-                    pattern, ', '.join(addresses)))
-                for a in addresses:
-                    emails.setdefault(a, '')
-                    emails[a] += error.format_for_display() + '\n'
-                break
+        project_name = error.get_project_name()
+        if project_name is not None:
+            addresses = cfg['project_notification_addresses'][project_name]
+            dbg('spec %s is targetted to "%s", mailing to %s' % (
+                    error.get_blueprint_name(), project_name,
+                    ', '.join(addresses)))
+            for a in addresses:
+                emails.setdefault(a, '')
+                emails[a] += error.format_for_display() + '\n'
         else:
-            print >> sys.stderr, error.format_for_display(), '(no error_contact pattern)'
+            print >> sys.stderr, error.format_for_display(), \
+                '(no error_contact pattern)'
 
     # send mails
     for addr, contents in emails.iteritems():
         msg = MIMEText(contents.encode('ascii', 'replace'))
         msg['Subject'] = 'Errors in work item definitions'
-        msg['From'] = 'Launchpad work item tracker <[e-mail]>'
+        msg['From'] = 'Launchpad work item tracker ' + \
+            '<[e-mail]>'
         msg['To'] = addr
         s = smtplib.SMTP()
         s.connect()
+<<<<<<< TREE
         s.sendmail('[e-mail]', addr, msg.as_string())
+=======
+        s.sendmail(os.environ.get(
+                       'EMAIL',
+                       pwd.getpwuid(os.geteuid()).pw_name + '@localhost'),
+                   addr,
+                   msg.as_string())
+>>>>>>> MERGE-SOURCE
         s.quit()
 
 
@@ -753,22 +840,29 @@
             bug_status_map[key] = unicode_or_None(val)
         bug_wi_states.update(bug_status_map)
 
-    lock_path = opts.database + ".collect_lock"
+    lock_path = opts.database + ".lock"
     lock_f = open(lock_path, "wb")
     if report_tools.lock_file(lock_f) is None:
         print "Another instance is already running"
         sys.exit(0)
 
     if "beta" in EDGE_SERVICE_ROOT:
-        lp = Launchpad.login_with('ubuntu-work-items', service_root=EDGE_SERVICE_ROOT.replace("edge.", "").replace("beta", "devel"))
+        service_root = EDGE_SERVICE_ROOT
+        service_root = service_root.replace("edge.", "")
+        service_root = service_root.replace("beta", "devel")
+        lp = Launchpad.login_with('ubuntu-work-items',
+                                  service_root=service_root)
     else:
-        lp = Launchpad.login_with('ubuntu-work-items', service_root="production", version="devel")
+        lp = Launchpad.login_with('ubuntu-work-items',
+                                  service_root="production", version="devel")
 
     store = get_store(opts.database)
     collector = CollectorStore(store, lp, error_collector)
 
     # reset status for current day
     collector.clear_todays_workitems()
+    # We can delete all blueprints while keeping work items for previous days
+    # because there's no foreign key reference from WorkItem to Blueprint.
     collector.clear_blueprints()
     collector.clear_metas()
     collector.clear_complexitys()

=== added file 'collect_jira'
--- collect_jira	1970-01-01 00:00:00 +0000
+++ collect_jira	2012-10-09 09:20:30 +0000
@@ -0,0 +1,229 @@
+#!/usr/bin/python
+#
+# Pull items from cards.linaro.org and put them into a database.
+
+import logging
+import optparse
+import os
+import simplejson
+import sys
+import urllib2
+
+import jira
+from lpworkitems.collect_roadmap import (
+    CollectorStore,
+    )
+from lpworkitems.database import get_store
+from lpworkitems.error_collector import (
+    ErrorCollector,
+    StderrErrorCollector,
+    )
+from lpworkitems.models_roadmap import (
+    Lane,
+    Card,
+    )
+import report_tools
+
+
+# An ErrorCollector to collect the data errors for later reporting
+error_collector = None
+
+
+logger = logging.getLogger("linarojira")
+
+JIRA_API_URL = 'http://cards.linaro.org/rest/api/2'
+JIRA_PROJECT_KEY = 'CARD'
+JIRA_ISSUE_BY_KEY_URL = 'http://cards.linaro.org/browse/%s'
+
+
+def dbg(msg):
+    '''Print out debugging message if debugging is enabled.'''
+    logger.debug(msg)
+
+
+def get_json_data(url):
+    data = None
+    try:
+        data = simplejson.load(urllib2.urlopen(url))
+    except urllib2.HTTPError, e:
+        print "HTTP error for url '%s': %d" % (url, e.code)
+    except urllib2.URLError, e:
+        print "Network error for url '%s': %s" % (url, e.reason.args[1])
+    except ValueError, e:
+        print "Data error for url '%s': %s" % (url, e.args[0])
+
+    return data
+
+
+def jira_import(collector, cfg, opts):
+    '''Collect roadmap items from JIRA into DB.'''
+
+    # import JIRA versions as database Lanes
+    result = jira.do_request(opts, 'project/%s/versions' % JIRA_PROJECT_KEY)
+    for version in result:
+        dbg('Adding lane (name = %s, id = %s)' %
+            (version['name'], version['id']))
+        model_lane = Lane(unicode(version['name']), int(version['id']))
+        if model_lane.name == cfg['current_lane']:
+            model_lane.is_current = True
+        else:
+            model_lane.is_current = False
+        collector.store_lane(model_lane)
+
+    # find id of "Sponsor" custom field in JIRA
+    result = jira.do_request(opts, 'field')
+    sponsor_fields = [field for field in result if field['name'] == 'Sponsor']
+    assert len(sponsor_fields) == 1, 'Not a single Sponsor field'
+    sponsor_field_id = sponsor_fields[0]['id']
+
+    # import JIRA issues as database Cards
+    result = jira.do_request(
+        opts, 'search', jql='project = %s' % JIRA_PROJECT_KEY,
+        fields=['summary', 'fixVersions', 'status', 'components',
+                'priority', 'description', 'timetracking', sponsor_field_id])
+    for issue in result['issues']:
+        fields = issue['fields']
+        name = unicode(fields['summary'])
+        card_id = int(issue['id'])
+        key = unicode(issue['key'])
+        fixVersions = fields['fixVersions']
+        if len(fixVersions) == 0:
+            dbg('Skipping card without lane (name = %s, key = %s)' %
+                (name, key))
+            continue
+        # JIRA allows listing multiple versions in fixVersions
+        assert len(fixVersions) == 1
+        lane_id = int(fixVersions[0]['id'])
+
+        dbg('Adding card (name = %s, id = %s, lane_id = %s, key = %s)' %
+                (name, card_id, lane_id, key))
+        model_card = Card(name, card_id, lane_id, key)
+        model_card.status = unicode(fields['status']['name'])
+        components = fields['components']
+        if len(components) == 0:
+            dbg('Skipping card without component (name = %s, key = %s)' %
+                (name, key))
+        # JIRA allows listing multiple components
+        assert len(components) == 1
+        model_card.team = unicode(components[0]['name'])
+        model_card.priority = unicode(fields['priority']['name'])
+        size_fields = []
+        timetracking = fields['timetracking']
+        if 'originalEstimate' in timetracking:
+            size_fields += [
+                'original estimate: %s' % timetracking['originalEstimate']]
+        if 'remainingEstimate' in timetracking:
+            size_fields += [
+                'remaining estimate: %s' % timetracking['remainingEstimate']]
+        model_card.size = unicode(', '.join(size_fields))
+        model_card.sponsor = u''
+        # None if no sponsor is selected
+        if fields[sponsor_field_id] is not None:
+            sponsors = [s['value'] for s in fields[sponsor_field_id]]
+            model_card.sponsor = unicode(', '.join(sorted(sponsors)))
+        model_card.url = JIRA_ISSUE_BY_KEY_URL % key
+        # XXX need to either download the HTML version or convert this to HTML
+        model_card.description = unicode(fields['description'])
+        # acceptance criteria is in the description
+        model_card.acceptance_criteria = u''
+        collector.store_card(model_card)
+    return
+
+########################################################################
+#
+# Program operations and main
+#
+########################################################################
+
+
+def parse_argv():
+    '''Parse CLI arguments.
+
+    Return (options, args) tuple.
+    '''
+    optparser = optparse.OptionParser()
+    optparser.add_option('-d', '--database',
+        help='Path to database', dest='database', metavar='PATH')
+    optparser.add_option('-c', '--config',
+        help='Path to configuration file', dest='config', metavar='PATH')
+    optparser.add_option('--debug', action='store_true', default=False,
+        help='Enable debugging output in parsing routines')
+    optparser.add_option('--mail', action='store_true', default=False,
+        help='Send data errors as email (according to "error_config" map in '
+            'config file) instead of printing to stderr', dest='mail')
+    optparser.add_option('--jira-username', default='robot',
+        help='JIRA username for authentication', dest='jira_username')
+    optparser.add_option('--jira-password', default='cuf4moh2',
+        help='JIRA password for authentication', dest='jira_password')
+
+    (opts, args) = optparser.parse_args()
+
+    if not opts.database:
+        optparser.error('No database given')
+    if not opts.config:
+        optparser.error('No config given')
+
+    return opts, args
+
+
+def setup_logging(debug):
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.INFO)
+    formatter = logging.Formatter("%(message)s")
+    ch.setFormatter(formatter)
+    logger.setLevel(logging.INFO)
+    logger.addHandler(ch)
+    if debug:
+        ch.setLevel(logging.DEBUG)
+        formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        ch.setFormatter(formatter)
+        logger.setLevel(logging.DEBUG)
+
+
+def update_todays_blueprint_daily_count_per_state(collector):
+    """Clear today's entries and create them again to reflect the current
+    state of blueprints."""
+    collector.clear_todays_blueprint_daily_count_per_state()
+    collector.store_roadmap_bp_count_per_state()
+
+
+def main():
+    report_tools.fix_stdouterr()
+
+    (opts, args) = parse_argv()
+    opts.jira_api_url = JIRA_API_URL
+
+    setup_logging(opts.debug)
+
+    global error_collector
+    if opts.mail:
+        error_collector = ErrorCollector()
+    else:
+        error_collector = StderrErrorCollector()
+
+    cfg = report_tools.load_config(opts.config)
+
+    lock_path = opts.database + ".lock"
+    lock_f = open(lock_path, "wb")
+    if report_tools.lock_file(lock_f) is None:
+        print "Another instance is already running"
+        sys.exit(0)
+
+    store = get_store(opts.database)
+    collector = CollectorStore(store, '', error_collector)
+
+    collector.clear_lanes()
+    collector.clear_cards()
+
+    jira_import(collector, cfg, opts)
+
+    update_todays_blueprint_daily_count_per_state(collector)
+
+    store.commit()
+
+    os.unlink(lock_path)
+
+
+if __name__ == '__main__':
+    main()

=== added file 'collect_roadmap'
--- collect_roadmap	1970-01-01 00:00:00 +0000
+++ collect_roadmap	2012-10-09 09:20:30 +0000
@@ -0,0 +1,301 @@
+#!/usr/bin/python
+#
+# Pull items from the Linaro roadmap in Kanbantool and put them into a database.
+
+import logging
+import optparse
+import os
+import simplejson
+import sys
+import urllib2
+
+from lpworkitems.collect_roadmap import (
+    CollectorStore,
+    get_json_item,
+    lookup_kanban_priority,
+    )
+from lpworkitems.database import get_store
+from lpworkitems.error_collector import (
+    ErrorCollector,
+    StderrErrorCollector,
+    )
+from lpworkitems.models_roadmap import (
+    Lane,
+    Card,
+    )
+from utils import unicode_or_None
+import report_tools
+
+
+# An ErrorCollector to collect the data errors for later reporting
+error_collector = None
+
+
+logger = logging.getLogger("linaroroadmap")
+
+
+def dbg(msg):
+    '''Print out debugging message if debugging is enabled.'''
+    logger.debug(msg)
+
+
+def get_kanban_url(item_url, api_token):
+    base_url = 'https://linaro.kanbantool.com/api/v1'
+    return "%s/%s?api_token=%s" % (base_url, item_url, api_token)
+
+
+def get_json_data(url):
+    data = None
+    try:
+        data = simplejson.load(urllib2.urlopen(url))
+    except urllib2.HTTPError, e:
+        print "HTTP error for url '%s': %d" % (url, e.code)
+    except urllib2.URLError, e:
+        print "Network error for url '%s': %s" % (url, e.reason.args[1])
+    except ValueError, e:
+        print "Data error for url '%s': %s" % (url, e.args[0])
+
+    return data
+
+
+def kanban_import_lanes(collector, workflow_stages, cfg):
+    nodes = {}
+    root_node_id = None
+    lanes_to_ignore = ['Legend']
+
+    # Iterate over all workflow_stages which may be in any order.
+    for workflow_stage in workflow_stages:
+        if workflow_stage['name'] in lanes_to_ignore:
+            dbg("Ignoring lane %s." % workflow_stage['name'])
+            continue
+        parent_id = workflow_stage['parent_id']
+        if parent_id is None:
+            assert root_node_id is None, 'We have already found the root node.'
+            root_node_id = workflow_stage['id']
+        else:
+            if parent_id not in nodes:
+                nodes[parent_id] = []
+            # Add child workflow_stage
+            nodes[parent_id].append(workflow_stage)
+
+    statuses = []
+    for node in nodes[root_node_id]:
+        assert node['parent_id'] == root_node_id
+        model_lane = Lane(get_json_item(node, 'name'),
+                          node['id'])
+        if model_lane.name == cfg['current_lane']:
+            model_lane.is_current = True
+        else:
+            model_lane.is_current = False
+        collector.store_lane(model_lane)
+        node_id = node['id']
+        if node_id in nodes:
+            statuses.extend(nodes[node_id])
+    return statuses
+
+
+def kanban_import_cards(collector, tasks, status_list, card_types, papyrs_token):
+    types_to_ignore = ['Summits']
+    for task in tasks:
+        dbg("Collecting card '%s'." % (task['task']['name']))
+        status_id = task['task']['workflow_stage_id']
+        assert status_id is not None
+        task_status = None
+        for status in status_list:
+            if status['id'] == status_id:
+                task_status = status
+                break
+        card_type_id = task['task']['card_type_id']
+        card_type_name = None
+        for card_type in card_types:
+            if card_type['id'] == card_type_id:
+                card_type_name = card_type['name']
+                break
+        else:
+            dbg("Cannot find type for card '%s'." % (task['task']['name']))
+        if card_type_name in types_to_ignore:
+            dbg("Ignoring card '%s' since it\'s type is '%s'." %  \
+                    (task['task']['name'], card_type['name']))
+        else:
+            if task_status is not None:
+                lane_id = task_status['parent_id']
+                assert lane_id is not None
+            else:
+                lane_id = status_id
+            if not collector.lane_is_collected(lane_id):
+                dbg("Ignoring card '%s' since it\'s Lane is ignored." %  \
+                        (task['task']['name']))
+                continue
+            model_card = Card(get_json_item(task['task'], 'name'),
+                              task['task']['id'], lane_id,
+                              get_json_item(task['task'], 'external_id'))
+            if task_status is not None:
+                model_card.status = get_json_item(task_status, 'name')
+            model_card.team = unicode_or_None(card_type_name)
+            model_card.priority = lookup_kanban_priority(
+                task['task']['priority'])
+            model_card.size = get_json_item(task['task'], 'size_estimate')
+            model_card.sponsor = get_json_item(task['task'],
+                                               'custom_field_1')
+
+            external_link = task['task']['custom_field_2']
+            if external_link is not None and external_link is not '':
+                model_card.url = unicode_or_None(external_link)
+                dbg('Getting Papyrs information from %s.' % external_link)
+                papyrs_data = papyrs_import(collector, external_link, papyrs_token)
+                model_card.description = get_json_item(papyrs_data,
+                                                       'description')
+                model_card.acceptance_criteria = get_json_item(
+                    papyrs_data, 'acceptance_criteria')
+            collector.store_card(model_card)
+
+
+def kanban_import(collector, cfg, board_id, api_token, papyrs_token):
+    '''Collect roadmap items from KanbanTool into DB.'''
+    board_url = get_kanban_url('boards/%s.json' % board_id, api_token)
+    board = get_json_data(board_url)
+    assert board is not None, "Could not access board %s." % board_id
+    card_types = board['board']['card_types']
+    status_list = kanban_import_lanes(collector,
+                                      board['board']['workflow_stages'], cfg)
+
+    tasks_url = get_kanban_url('boards/%s/tasks.json' % board_id, api_token)
+    tasks = get_json_data(tasks_url)
+    kanban_import_cards(collector, tasks, status_list, card_types, papyrs_token)
+
+
+def papyrs_import(collector, requirement_url, papyrs_token):
+    description = None
+    acceptance_criteria = None
+
+    page = get_json_data(requirement_url + '?json&auth_token=%s' % papyrs_token)
+    if page is None:
+        return {'description': None,
+                'acceptance_criteria': None}
+
+    page_text_items = page[0]
+    page_extra_items = page[1]
+
+    has_found_description = False
+    last_heading = ''
+    for page_item in page_text_items:
+        if page_item['classname'] == 'Heading':
+            last_heading = page_item['text']
+        if page_item['classname'] == 'Paragraph':
+            if not has_found_description:
+                description = page_item['html']
+                has_found_description = True
+            elif 'Acceptance Criteria' in last_heading:
+                acceptance_criteria = page_item['html']
+
+    return {'description': get_first_paragraph(description),
+            'acceptance_criteria': get_first_paragraph(acceptance_criteria)}
+
+
+def get_first_paragraph(text):
+    if text is None:
+        return None
+    # This might break, depending on what type of line breaks
+    # whoever authors the Papyrs document uses.
+    first_pararaph, _, _ = text.partition('<br>')
+    return first_pararaph
+
+
+########################################################################
+#
+# Program operations and main
+#
+########################################################################
+
+def parse_argv():
+    '''Parse CLI arguments.
+
+    Return (options, args) tuple.
+    '''
+    optparser = optparse.OptionParser()
+    optparser.add_option('-d', '--database',
+        help='Path to database', dest='database', metavar='PATH')
+    optparser.add_option('-c', '--config',
+        help='Path to configuration file', dest='config', metavar='PATH')
+    optparser.add_option('--debug', action='store_true', default=False,
+        help='Enable debugging output in parsing routines')
+    optparser.add_option('--mail', action='store_true', default=False,
+        help='Send data errors as email (according to "error_config" map in '
+            'config file) instead of printing to stderr', dest='mail')
+    optparser.add_option('--board',
+        help='Board id at Kanbantool', dest='board')
+    optparser.add_option('--kanbantoken',
+        help='Kanbantool API token for authentication', dest='kanban_token')
+    optparser.add_option('--papyrstoken',
+        help='Papyrs API token for authentication', dest='papyrs_token')
+
+    (opts, args) = optparser.parse_args()
+
+    if not opts.database:
+        optparser.error('No database given')
+    if not opts.config:
+        optparser.error('No config given')
+
+    return opts, args
+
+
+def setup_logging(debug):
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.INFO)
+    formatter = logging.Formatter("%(message)s")
+    ch.setFormatter(formatter)
+    logger.setLevel(logging.INFO)
+    logger.addHandler(ch)
+    if debug:
+        ch.setLevel(logging.DEBUG)
+        formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        ch.setFormatter(formatter)
+        logger.setLevel(logging.DEBUG)
+
+
+def update_todays_blueprint_daily_count_per_state(collector):
+    """Clear today's entries and create them again to reflect the current
+    state of blueprints."""
+    collector.clear_todays_blueprint_daily_count_per_state()
+    collector.store_roadmap_bp_count_per_state()
+
+
+def main():
+    report_tools.fix_stdouterr()
+
+    (opts, args) = parse_argv()
+
+    setup_logging(opts.debug)
+
+    global error_collector
+    if opts.mail:
+        error_collector = ErrorCollector()
+    else:
+        error_collector = StderrErrorCollector()
+
+    cfg = report_tools.load_config(opts.config)
+
+    lock_path = opts.database + ".lock"
+    lock_f = open(lock_path, "wb")
+    if report_tools.lock_file(lock_f) is None:
+        print "Another instance is already running"
+        sys.exit(0)
+
+    store = get_store(opts.database)
+    collector = CollectorStore(store, '', error_collector)
+
+    collector.clear_lanes()
+    collector.clear_cards()
+
+    kanban_import(collector, cfg, opts.board, opts.kanban_token, opts.papyrs_token)
+
+    update_todays_blueprint_daily_count_per_state(collector)
+
+    store.commit()
+
+    os.unlink(lock_path)
+
+
+if __name__ == '__main__':
+    main()

=== modified file 'css/status.css'
--- css/status.css	2011-05-18 20:51:50 +0000
+++ css/status.css	2012-10-09 09:20:30 +0000
@@ -46,6 +46,64 @@
     font-size: 1.2em;
 }
 
+
+.roadmap_progress_text {
+    position: absolute;
+    top:0; left:0;
+
+    padding-top: 5px;
+
+    color: #ffffff;
+    text-align: center;
+    width: 100%;
+}
+
+.roadmap_wrap {
+    border: 1px solid black;
+    position: relative;
+    margin-top: 2px;
+    margin-bottom: 3px;
+    margin-left: auto;
+    margin-right: auto;
+    background-color: #bdbdbd;
+}
+
+.roadmap_wrap, .roadmap_value {
+    width: 300px;
+    height: 28px;
+}
+
+table .roadmap_wrap, table .roadmap_value {
+    border: 0px;
+    width: 155px;
+    height: 1.4em;
+    background-color: #ffffff;
+}
+
+.roadmap_value {
+    float: left;
+}
+
+.roadmap_value .Completed {
+    background-color: green;
+    height: inherit
+}
+
+.roadmap_value .Blocked {
+    background-color: red;
+    height: inherit
+}
+
+.roadmap_value .InProgress {
+    background-color: gray;
+    height: inherit
+}
+
+.roadmap_value .Planned {
+    background-color: orange;
+    height: inherit
+}
+
 .progress_wrap {
     position: relative;
     border: 1px solid black;

=== modified file 'generate-all'
--- generate-all	2011-09-09 07:14:11 +0000
+++ generate-all	2012-10-09 09:20:30 +0000
@@ -6,7 +6,7 @@
 # Copyright (C) 2010, 2011 Canonical Ltd.
 # License: GPL-3
 
-import optparse, os.path, sys
+import optparse, os.path, sys, errno
 
 import report_tools
 
@@ -47,13 +47,7 @@
     burnup_chart_teams = []
     primary_team = None
 
-lock_path = opts.database + ".generate_lock"
-lock_f = open(lock_path, "wb")
-if report_tools.lock_file(lock_f) is None:
-    print "Another instance is already running"
-    sys.exit(0)
-
-lock_path = opts.database + ".generate_lock"
+lock_path = opts.database + ".lock"
 lock_f = open(lock_path, "wb")
 if report_tools.lock_file(lock_f) is None:
     print "Another instance is already running"
@@ -80,14 +74,60 @@
 usersubdir = os.path.join(opts.output_dir, 'u')
 try:
     os.mkdir(usersubdir)
-except OSError:
-    None
+except OSError as exc:
+    if exc.errno == errno.EEXIST:
+        pass
+    else:
+        raise
 
 groupssubdir = os.path.join(opts.output_dir, 'group')
 try:
     os.mkdir(groupssubdir)
-except OSError:
-    None
+except OSError as exc:
+    if exc.errno == errno.EEXIST:
+        pass
+    else:
+        raise
+
+lanessubdir = os.path.join(opts.output_dir, '..', 'lane')
+try:
+    os.mkdir(lanessubdir)
+except OSError as exc:
+    if exc.errno == errno.EEXIST:
+        pass
+    else:
+        raise
+
+cardssubdir = os.path.join(opts.output_dir, '..', 'card')
+try:
+    os.mkdir(cardssubdir)
+except OSError as exc:
+    if exc.errno == errno.EEXIST:
+        pass
+    else:
+        raise
+
+# roadmap lanes
+for lane in report_tools.lanes(store):
+    basename = os.path.join(lanessubdir, lane.name)
+    report_tools.roadmap_pages(my_path, opts.database, basename, opts.config,
+                               lane, root=opts.root)
+
+# roadmap cards
+for card in report_tools.cards(store):
+    if card.roadmap_id != '':
+        page_name = card.roadmap_id
+    else:
+        page_name = str(card.card_id)
+    basename = os.path.join(cardssubdir, page_name)
+    report_tools.roadmap_cards(my_path, opts.database, basename, opts.config,
+                               card, root=opts.root)
+
+# roadmap front page
+basename = os.path.join(lanessubdir, 'index')
+lane = report_tools.current_lane(store)
+report_tools.roadmap_pages(my_path, opts.database, basename, opts.config,
+                           lane, root=opts.root)
 
 for u in users:
     for m in milestones:
@@ -156,11 +196,10 @@
     basename = os.path.join(opts.output_dir, status)
     report_tools.workitem_list(my_path, opts.database, basename, opts.config, status, root=opts.root)
 
-# front page
+# cycle front page
 basename = os.path.join(opts.output_dir, 'index')
 report_tools.status_overview(my_path, opts.database, basename, opts.config, root=opts.root)
 
-
 def copy_files(source_dir):
     for filename in os.listdir(source_dir):
         dest = open(os.path.join(opts.output_dir, filename), 'w')

=== modified file 'html-report'
--- html-report	2012-06-20 19:54:52 +0000
+++ html-report	2012-10-09 09:20:30 +0000
@@ -10,6 +10,13 @@
 
 from report_tools import escape_url
 import report_tools
+from roadmap_health import (
+    card_health_checks,
+)
+from lpworkitems.models import (
+    ROADMAP_STATUSES_MAP,
+    ROADMAP_ORDERED_STATUSES,
+)
 
 
 class WorkitemTarget(object):
@@ -453,6 +460,97 @@
         print report_tools.fill_template(
             "workitem_list.html", data, theme=opts.theme)
 
+    def roadmap_page(self, store, opts):
+        if opts.lane is None:
+            print "<h1>Error, no lane specified.</h1>"
+        if not opts.title:
+            title = opts.lane
+        else:
+            title = opts.title
+
+        data = self.template_data(store, opts)
+        lane = report_tools.lane(store, opts.lane)
+        lanes = report_tools.lanes(store)
+        statuses = []
+        bp_status_totals = {'Completed': 0, 'Total': 0, 'Percentage': 0}
+        for status, cards in report_tools.statuses(store, lane):
+            cards_with_bps = []
+            for card in cards:
+                report_tools.check_card_health(store, card_health_checks, card)
+                blueprint_status_counts = report_tools.card_bp_status_counts(
+                    store, card.roadmap_id)
+                total = sum(blueprint_status_counts.values())
+                bp_percentages = dict.fromkeys(ROADMAP_ORDERED_STATUSES, 0)
+                bp_status_totals['Completed'] += \
+                    blueprint_status_counts['Completed']
+                for key in ROADMAP_STATUSES_MAP:
+                    bp_status_totals['Total'] += blueprint_status_counts[key]
+                    if total > 0:
+                        bp_percentages[key] = (
+                            100.0 * blueprint_status_counts[key] / total)
+
+                cards_with_bps.append({'card': card,
+                                       'bp_statuses': blueprint_status_counts,
+                                       'bp_percentages': bp_percentages})
+            statuses.append(dict(name=status, cards=cards_with_bps))
+        if bp_status_totals['Total'] > 0:
+            bp_status_totals['Percentage'] = (100 * bp_status_totals['Completed'] /
+                bp_status_totals['Total'])
+
+        data.update(dict(statuses=statuses))
+        data.update(dict(bp_status_totals=bp_status_totals))
+        data.update(dict(status_order=ROADMAP_ORDERED_STATUSES))
+        data.update(dict(page_type="roadmap_lane"))
+        data.update(dict(lane_title=title))
+        data.update(dict(lanes=lanes))
+        data.update(dict(chart_url=opts.chart_url))
+        print report_tools.fill_template(
+            "roadmap_lane.html", data, theme=opts.theme)
+
+
+    def roadmap_card(self, store, opts):
+        if opts.card is None:
+            print "<h1>Error, no card specified.</h1>"
+
+        data = self.template_data(store, opts)
+        card = report_tools.card(store, int(opts.card)).one()
+        health_checks = report_tools.check_card_health(store, card_health_checks, card)
+        lane = report_tools.lane(store, None, id=card.lane_id)
+
+        if not opts.title:
+            title = card.name
+        else:
+            title = opts.title
+
+        blueprints = report_tools.card_blueprints_by_status(store, card.roadmap_id)
+        bp_status_totals = {'Completed': 0, 'Total': 0, 'Percentage': 0}
+        bp_status_totals['Total'] = (len(blueprints['Planned']) +
+                                     len(blueprints['Blocked']) +
+                                     len(blueprints['In Progress']) +
+                                     len(blueprints['Completed']))
+        bp_status_totals['Completed'] = len(blueprints['Completed'])
+        if bp_status_totals['Total'] > 0:
+            bp_status_totals['Percentage'] = (100 * bp_status_totals['Completed'] /
+                bp_status_totals['Total'])
+
+        card_has_blueprints = bp_status_totals['Total'] > 0
+
+        status_order = ROADMAP_ORDERED_STATUSES[:]
+        status_order.reverse()
+
+        data.update(dict(page_type="roadmap_card"))
+        data.update(dict(card_title=title))
+        data.update(dict(card=card))
+        data.update(dict(health_checks=health_checks))
+        data.update(dict(lane=lane.name))
+        data.update(dict(status_order=status_order))
+        data.update(dict(blueprints=blueprints))
+        data.update(dict(bp_status_totals=bp_status_totals))
+        data.update(dict(card_has_blueprints=card_has_blueprints))
+
+        print report_tools.fill_template(
+            "roadmap_card.html", data, theme=opts.theme)
+
 
 class WorkitemsOnDate(object):
 
@@ -531,6 +629,10 @@
             help="Include all milestones targetted to this date.")
     optparser.add_option('--theme', dest="theme",
             help="The theme to use.", default="linaro")
+    optparser.add_option('--lane',
+        help='Roadmap lane', dest='lane')
+    optparser.add_option('--card',
+        help='Roadmap card', dest='card')
 
     (opts, args) = optparser.parse_args()
     if not opts.database:

=== added file 'jira.py'
--- jira.py	1970-01-01 00:00:00 +0000
+++ jira.py	2012-10-09 09:20:30 +0000
@@ -0,0 +1,55 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+import base64
+import optparse
+import simplejson
+import urllib2
+
+
+def do_request(opts, relpathname, **kwargs):
+    request = urllib2.Request('%s/%s' % (opts.jira_api_url, relpathname))
+    if opts.jira_username and opts.jira_password:
+        base64string = base64.encodestring(
+            '%s:%s' % (opts.jira_username, opts.jira_password)
+            ).replace('\n', '')
+        request.add_header('Authorization', 'Basic %s' % base64string)
+    request_data = None
+    if kwargs.keys():
+        request.add_header('Content-Type', 'application/json')
+        request_data = simplejson.dumps(kwargs)
+    response_data = urllib2.urlopen(request, request_data)
+    return simplejson.load(response_data)
+
+
+def main():
+    parser = optparse.OptionParser(usage="%prog")
+    parser.add_option("--jira-api-url", dest="jira_api_url",
+        default="http://cards.linaro.org/rest/api/2")
+    parser.add_option("--jira-username", dest="jira_username",
+        default="robot")
+    parser.add_option("--jira-password", dest="jira_password",
+        default="cuf4moh2")
+    opts, args = parser.parse_args()
+
+    # simple search
+    print do_request(opts, 'search', maxResults=1, jql='project = CARD',
+                     fields=['summary', 'status'])
+
+    # information about a project
+    #print do_request(opts, 'project/CARD')
+
+    # on creating issues
+    #print do_request(opts, 'issue/createmeta?projectIds=10000')
+
+    # on statuses
+    #print do_request(opts, 'status')
+
+    # on fields
+    #print do_request(opts, 'field')
+
+    # on a security level
+    #print do_request(opts, 'securitylevel/10000')
+
+if __name__ == "__main__":
+    main()

=== added file 'kanban-papyrs-to-jira'
--- kanban-papyrs-to-jira	1970-01-01 00:00:00 +0000
+++ kanban-papyrs-to-jira	2012-10-09 09:20:30 +0000
@@ -0,0 +1,397 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+# Copyright (C) 2012 Linaro Ltd.
+# Author: Loïc Minier <[e-mail]>
+# License: GPL-3
+
+import jira
+
+from bs4 import BeautifulSoup
+import logging
+import optparse
+import os
+import re
+import simplejson
+import sys
+import urllib2
+
+logger = logging.getLogger("linaroroadmap")
+
+def dbg(msg):
+    '''Print out debugging message if debugging is enabled.'''
+    logger.debug(msg)
+
+class InMemCollector:
+    def __init__(self):
+        self.lanes = []
+        self.cards = []
+
+    def store_lane(self, lane):
+        self.lanes.append(lane)
+
+    def store_card(self, card):
+        self.cards.append(card)
+
+    def lane_is_collected(self, lane_id):
+        for l in self.lanes:
+            if l.lane_id == lane_id:
+                return True
+        return False
+
+def kanban_request(opts, relpathname, method='GET', **kwargs):
+    request = urllib2.Request(
+        '%s/%s.json?_m=%s' % (opts.kanban_api_url, relpathname, method))
+    if opts.kanban_token:
+        request.add_header('X-KanbanToolToken', opts.kanban_token)
+    request_data = None
+    if kwargs.keys():
+        request.add_header('Content-Type', 'application/json')
+        request_data = simplejson.dumps(kwargs)
+        print request_data
+    response_data = urllib2.urlopen(request, request_data)
+    return simplejson.load(response_data)
+
+def get_papyrs_page(papyrs_url, token):
+    url = '%s?json&auth_token=%s' % (papyrs_url, token)
+    return simplejson.load(urllib2.urlopen(url))
+
+def get_kanban_boards(opts):
+    return kanban_request(opts, 'boards')
+
+def get_kanban_board(opts, board_id):
+    return kanban_request(opts, 'boards/%s' % board_id)
+
+def get_kanban_tasks(opts, board_id):
+    return kanban_request(opts, 'boards/%s/tasks' % board_id)
+
+def get_kanban_task(opts, board_id, task_id):
+    return kanban_request(opts, 'boards/%s/tasks/%s' % (board_id, task_id))
+
+def put_kanban_task(opts, board_id, task_id, **kwargs):
+    return kanban_request(opts, 'boards/%s/tasks/%s' % (board_id, task_id), method='PUT', **kwargs)
+
+def main():
+    # TODO: add support for passing a card id or papyrs URL
+    parser = optparse.OptionParser(usage="%prog")
+    parser.add_option("--kanban-api-url", dest="kanban_api_url",
+        default="https://linaro.kanbantool.com/api/v1")
+    parser.add_option("--jira-api-url", dest="jira_api_url",
+        default="http://cards.linaro.org/rest/api/2")
+    # defaults are read-only ~linaro-infrastructure tokens
+    parser.add_option("--kanban-token", dest="kanban_token",
+        default="9F209W7Y84TE")
+    parser.add_option("--papyrs-token", dest="papyrs_token",
+        default="868e9088b53c")
+    parser.add_option("--jira-username", dest="jira_username",
+        default="robot")
+    parser.add_option("--jira-password", dest="jira_password",
+        default="cuf4moh2")
+    parser.add_option("--jira-project", dest="jira_project_name",
+        default="CARD")
+    parser.add_option("--jira-issuetype", dest="jira_issuetype_name",
+        default="Roadmap Card")
+    parser.add_option("--board-id", dest="board_id", default="10721")
+    parser.add_option('--debug', action='store_true', default=True,
+        help='Enable debugging output in parsing routines')
+    parser.add_option('--board',
+        help='Board id at Kanban Tool', dest='board', default='10721')
+    opts, args = parser.parse_args()
+
+    if os.environ.get("DEBUG", None) is not None:
+        opts.debug = True
+
+    if len(args) != 0:
+        parser.error("You can not pass any argument")
+
+    if opts.kanban_token is None:
+        sys.stderr.write("No Kanbantool API token given")
+    if opts.papyrs_token is None:
+        sys.stderr.write("No Papyrs API token given")
+
+    # logging setup
+    logger = logging.getLogger()
+    ch = logging.StreamHandler()
+    formatter = logging.Formatter("%(asctime)s %(message)s")
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+    if opts.debug:
+        logger.setLevel(logging.DEBUG)
+
+    boards = get_kanban_boards(opts)
+    # dump
+    for board in boards:
+        board = board['board']
+        dbg('Found board "%s" with id %s' % (board['name'], board['id']))
+    dbg('')
+
+    assert 1 == len(filter(lambda b: str(b['board']['id']) == opts.board_id, boards)), \
+        'Expected exactly one board with id %s' % opts.board_id
+
+    board = get_kanban_board(opts, opts.board_id)
+    board = board['board']
+
+    workflow_stages = board['workflow_stages']
+    # ideally order wouldn't matter but the "position" field of our workflow stages
+    # is bogus (always 1) so we can't use it
+    leaf_workflow_stages = []
+    for workflow_stage in workflow_stages:
+        childs = filter(
+            lambda ws: ws['parent_id'] == workflow_stage['id'], workflow_stages)
+        if not childs:
+            # build a name list for leaf workflow stages
+            name = []
+            id = workflow_stage['id']
+            while True:
+                ws = filter(lambda ws: ws['id'] == id, workflow_stages)[0]
+                if ws['name'] is None:
+                    break
+                name = [ws['name']] + name
+                id = ws['parent_id']
+            leaf_workflow_stages.append((workflow_stage['id'], name))
+    # dump
+    for id, name in leaf_workflow_stages:
+        pretty_name = "/".join(name)
+        dbg('Found leaf workflow stage %s with id %s' % (pretty_name, id))
+    dbg('')
+
+    card_types = board['card_types']
+    # dump
+    for card_type in card_types:
+        dbg('Found card type %s with id %s' % (card_type['name'], card_type['id']))
+    dbg('')
+
+    def get_leaf_workflow_stage_name(worfklow_stage_id):
+        return [name
+                    for id, name
+                    in leaf_workflow_stages
+                    if id == worfklow_stage_id][0]
+
+    def get_card_type_name(card_type_id):
+        return [card_type['name']
+                    for card_type
+                    in card_types
+                    if card_type['id'] == card_type_id][0]
+
+    def filter_tasks(task):
+        # ignore tasks in Legend and Deferred workflow stages
+        lwsn = get_leaf_workflow_stage_name(task['workflow_stage_id'])
+        if lwsn in (['Legend'], ['Deferred']):
+            dbg('Ignoring task %s in workflow stage %s'
+                    % (task['external_id'], "/".join(lwsn)))
+            return False
+        # ignore tasks with Summit and Unknown card type names
+        card_type_name = get_card_type_name(task['card_type_id'])
+        if card_type_name in ('Summits', 'Unknown'):
+            dbg('Ignoring task %s with card type name %s'
+                    % (task['external_id'], card_type_name))
+            return False
+        return True
+
+    tasks = get_kanban_tasks(opts, opts.board_id)
+    tasks = [t['task'] for t in tasks if filter_tasks(t['task'])]
+    # dump
+    for task in tasks:
+        dbg('Found task %s with id %s, workflow_stage_id %s, priority %s, '
+            'card_type_id %s, custom_field_2 %s, and external_id %s'
+                % (task['name'], task['id'], task['workflow_stage_id'],
+                   task['priority'], task['card_type_id'],
+                   task['custom_field_2'], task['external_id']))
+
+    CARD_TYPE_NAMES_TO_PREFIXES = {
+        'LAVA': 'LAVA',
+        'Android': 'ANDROID',
+        'Linux & Ubuntu': 'LINUX',
+        'TCWG': 'TCWG',
+        'GWG': 'GWG',
+        'MMWG': 'MMWG',
+        'KWG': 'KWG',
+        'PMWG': 'PMWG',
+        'OCTO': 'OCTO',
+    }
+
+    # check consistency of external_id with external_link and custom_field_2
+    # (papyrs URL), and of external_id with card_type name
+    for task in tasks:
+        external_id = task['external_id']
+        papyrs_url = task['custom_field_2']
+        external_link = task['external_link']
+        assert papyrs_url == 'https://linaro.papyrs.com/%s' % external_id, \
+            'Incorrect papyrs URL %s for task %s' % (papyrs_url, external_id)
+        assert external_link == 'http://status.linaro.org/card/%s' % external_id, \
+            'Incorrect external_link %s for task %s' % (external_link, external_id)
+        card_type_name = get_card_type_name(task['card_type_id'])
+        prefix = CARD_TYPE_NAMES_TO_PREFIXES[card_type_name]
+        assert external_id.startswith(prefix), \
+            'Incorrect card type prefix %s for task %s' % (prefix, external_id)
+
+    # verify papyrs pages
+    #for task in tasks:
+    for task in []:
+        external_id = task['external_id']
+        papyrs_url = task['custom_field_2']
+        dbg('Fetching card %s' % task['name'])
+        papyrs_json = get_papyrs_page(papyrs_url, opts.papyrs_token)
+
+        try:
+            # number of columns
+            ncols = len(papyrs_json)
+            assert ncols == 2, 'Expected exactly two columns but got %s' % len(ncols)
+
+            # first column
+            col0 = papyrs_json[0]
+            p0 = col0[0]
+            classname = p0['classname']
+            assert classname == 'Heading', \
+                "First paragraph of first column should be a a heading but is %s" % classname
+            assert p0['text'] == p0['html'], \
+                "Expected text (%s) and HTML (%s) to be identical for first heading" % (p0['text'], p0['html'])
+            assert p0['text'] == task['name'], \
+                'Mismatch between first heading (%s) and task (%s)' % (p0['text'], task['name'])
+            for p in col0[1:-2]:
+                assert p['classname'] in ('Heading', 'Paragraph'), \
+                    'Got unexpected classname %s' % p['classname']
+                if p['classname'] == 'Heading':
+                    assert p['text'] == p['html'], \
+                        'Expected heading HTML (%s) to match text (%s)' % (p['html'], p['text'])
+                if p['classname'] == 'Paragraph':
+                    soup = BeautifulSoup('<root>%s</root>' % p['html'], 'xml')
+                    for tag in soup.root.find_all(True):
+                        assert tag.name in ('font', 'b', 'a', 'ul', 'ol', 'li', 'br', 'p', 'span', 'div', 'u'), 'Unexpected tag %s' % tag.name
+
+            # second column
+            pm1 = col0[-1]
+            assert pm1['classname'] == 'Discuss', \
+                'Expect last classname to be Discuss but got %s' % pm1['classname']
+
+            col1 = papyrs_json[1]
+            skip_next_paragraph = False
+            nattachs = 0
+            for p in col1:
+                if p['classname'] in ('Checklist', 'Twitters', 'Navigation'):
+                    pass
+                elif p['classname'] == 'Attachment':
+                    nattachs += 1
+                elif p['classname'] == 'Heading' and p['text'] == 'Attachments':
+                    pass
+                elif p['classname'] == 'Heading' and p['text'] == 'Metadata':
+                    skip_next_paragraph = True
+                elif p['classname'] == 'Paragraph' and skip_next_paragraph:
+                    skip_next_paragraph = False
+                else:
+                    assert False, 'Unexpected paragraph %s' % p
+            if nattachs > 0:
+                dbg('Found %s attachment(s) on card %s' % (nattachs, task['name']))
+        except Exception, e:
+            dbg(e)
+
+    # query jira data
+    jira_project_result = jira.do_request(opts, 'project/%s' % opts.jira_project_name)
+    jira_statuses_result = jira.do_request(opts, 'status')
+    jira_fields_result = jira.do_request(opts, 'field')
+    # not allowed
+    #jira_securitylevels_result = jira.do_request(opts, 'securitylevel')
+    jira_priorities_result = jira.do_request(opts, 'priority')
+
+    def search_jira_id(jira_result, name):
+        return [r['id'] for r in jira_result if r['name'] == name][0]
+
+    # http://cards.linaro.org/rest/api/2/project/CARD has id 10000
+    #jira_project_id = 10000
+    jira_project_id = jira_project_result['id']
+    # issuetype for "Roadmap Card" http://cards.linaro.org/rest/api/2/issuetype/9
+    #jira_issuetype_id = 9
+    jira_issuetype_id = search_jira_id(jira_project_result['issueTypes'], opts.jira_issuetype_name)
+    dbg('Found id %s for %s issueType' % (jira_issuetype_id, opts.jira_issuetype_name))
+    for component in jira_project_result['components']:
+        dbg('Found component %s with id %s' % (component['name'], component['id']))
+    for version in jira_project_result['versions']:
+        dbg('Found version %s with id %s' % (version['name'], version['id']))
+    for status in jira_statuses_result:
+        dbg('Found status %s with id %s' % (status['name'], status['id']))
+
+    TYPE_TO_COMPONENT = {
+        'LAVA': 'LAVA',
+        'Android': 'Android',
+        'Linux & Ubuntu': 'Linux & Ubuntu',
+        'TCWG': 'Toolchain WG',
+        'GWG': 'Graphics WG',
+        'MMWG': 'Multimedia WG',
+        'KWG': 'Kernel WG',
+        'PMWG': 'Power Management WG',
+        'OCTO': 'OCTO',
+    }
+
+    STAGE_TO_STATUS = {
+        'New/Draft': 'New/Drafting',
+        'New/Needs Work': 'New/Drafting',
+        'New/TSC Reviewed': 'New/Reviewed',
+        '2012Q1/Done': 'Approved',
+        '2012Q1/Ready': 'Approved',
+        '2012Q2/Forecast': 'Approved',
+        '2012Q3/Forecast': 'Approved',
+        '2012H2/Forecast': 'Approved',
+        '2013/Forecast': 'Approved',
+    }
+
+    STAGE_TO_VERSION = {
+        'New/Draft': None,
+        'New/Needs Work': None,
+        'New/TSC Reviewed': None,
+        '2012Q1/Done': '2012Q1',
+        '2012Q1/Ready': '2012Q1',
+        '2012Q2/Forecast': '2012Q2',
+        '2012Q3/Forecast': '2012Q3',
+        '2012H2/Forecast': '2012H2',
+        '2013/Forecast': '2013',
+    }
+
+    PRIORITY_MAP = {
+        -1: 'Minor',
+        0: 'Major',
+        1: 'Critical',
+    }
+
+    # actual copy
+    for task in tasks:
+        print task['name']
+        print task['external_id']
+        external_id = task['external_id']
+        papyrs_url = task['custom_field_2']
+        papyrs_json = get_papyrs_page(papyrs_url, opts.papyrs_token)
+        # first column
+        col0 = papyrs_json[0]
+        p0 = col0[0]
+        # assemble HTML of description
+        html = ""
+        for p in col0[1:-1]:
+            if p['classname'] == 'Heading':
+                html += '<h1>%s</h1>\n' % p['text']
+            if p['classname'] == 'Paragraph':
+                html += '%s\n' % p['html']
+        html = '{html}\n%s{html}\n' % html
+
+        stage = "/".join(get_leaf_workflow_stage_name(task['workflow_stage_id']))
+        status = STAGE_TO_STATUS[stage]
+        version = STAGE_TO_VERSION[stage]
+        type_name = get_card_type_name(task['card_type_id'])
+        component = TYPE_TO_COMPONENT[type_name]
+        priority = PRIORITY_MAP[task['priority']]
+
+        fields = {'project': {'id': jira_project_id},
+                  'summary': task['name'],
+                  'issuetype': {'id': jira_issuetype_id},
+                  'description': html,
+                  'components': [{'id': search_jira_id(jira_project_result['components'], component)}],
+                  search_jira_id(jira_fields_result, 'Alias Card ID'): task['external_id'],
+                  # XXX hardcoded default security level; also, can't set security level to Public via API
+                  #'security': {'id': search_jira_id(jira_securitylevels_result, 'Public')},
+                  'priority': {'id': search_jira_id(jira_priorities_result, priority)},
+                 }
+                  #'status': search_jira_id(jira_statuses_result, status),
+        if version:
+            fields['fixVersions'] = [{'id': search_jira_id(jira_project_result['versions'], version)}]
+        dbg('Uploading card %s' % fields)
+        print jira.do_request(opts, 'issue', fields=fields)
+
+if __name__ == "__main__":
+    main()

=== modified file 'lpworkitems/collect.py'
--- lpworkitems/collect.py	2011-12-06 15:20:43 +0000
+++ lpworkitems/collect.py	2012-10-09 09:20:30 +0000
@@ -26,6 +26,7 @@
 # "interesting")
 workitem_precedence = [None, u'done', u'postponed', u'blocked', u'todo', u'inprogress']
 
+
 class PersonCache(object):
     """A cache of Launchpad accounts."""
 
@@ -91,31 +92,29 @@
         project_name = self.lp.load(milestone.target.self_link).name
         existing_milestone = self.store.find(
             models.Milestone,
-            models.Milestone.name==milestone_name).any()
+            models.Milestone.name == milestone_name).any()
         if existing_milestone is not None:
-            # We only store a milestone for the first project that we
-            # see it in.
-            # Check that the dates match, otherwise it's very confusing
+            # TODO: We now allow for the same milestone in different projects
+            # to have different due dates (within reasonable limits).
+            # However, the old algorithm which relied on all due dates to match,
+            # only stores a single milestone with due date for the first project
+            # that we see it in (essentially, random one). This is expected to
+            # be elaborated shortly. Then this block can be removed completely,
+            # until then it is left as a reminder.
             target_date = None
             if milestone.date_targeted is not None:
-                target_date = milestone.date_targeted.strftime("%Y-%m-%d")
+                target_date = milestone.date_targeted.strftime("%Y-%m")
             existing_target_date = existing_milestone.due_date
             if isinstance(existing_target_date, datetime.date):
-                existing_target_date = existing_target_date.strftime("%Y-%m-%d")
+                existing_target_date = existing_target_date.strftime("%Y-%m")
             if (target_date and existing_target_date != target_date):
-                error = MilestoneError(
-                    milestone,
-                    "Milestone %s (%s) has due_date %s but %s already has "
-                    "the due date as %s" % (milestone.name, project_name,
-                        target_date, existing_milestone.project,
-                        existing_target_date))
-                self.error_collector.store_error(error)
-            return
-        db_milestone = models.Milestone()
-        db_milestone.name = milestone_name
-        db_milestone.due_date = milestone.date_targeted
-        db_milestone.project = project_name
-        self.store.add(db_milestone)
+                existing_milestone.due_date = milestone.date_targeted
+        else:
+            db_milestone = models.Milestone()
+            db_milestone.name = milestone_name
+            db_milestone.due_date = milestone.date_targeted
+            db_milestone.project = project_name
+            self.store.add(db_milestone)
 
     def store_lp_milestones(self, milestones):
         if self.store.find(models.Milestone).any() is not None:
@@ -270,7 +269,7 @@
             if ']' in desc:
                 off = desc.index(']')
                 assignee_name = desc[1:off]
-                desc = desc[off+1:].strip()
+                desc = desc[off + 1:].strip()
             else:
                 self.error_collector.record_blueprint_error(
                     self.blueprint,
@@ -314,6 +313,7 @@
     def get_workitem_if_tracked(self, task, projects=None,
                                 distro_release=None):
         target = self.lp.load(task.target.self_link)
+
         def get_rtype(obj):
             return urllib.splittag(obj.resource_type_link)[1]
         rtype = get_rtype(target)

=== added file 'lpworkitems/collect_roadmap.py'
--- lpworkitems/collect_roadmap.py	1970-01-01 00:00:00 +0000
+++ lpworkitems/collect_roadmap.py	2012-10-09 09:20:30 +0000
@@ -0,0 +1,71 @@
+import datetime
+
+from lpworkitems import models_roadmap
+from utils import unicode_or_None
+
+
+class CollectorStore(object):
+
+    def __init__(self, store, base_url, error_collector):
+        self.store = store
+        self.base_url = base_url
+        self.error_collector = error_collector
+
+    def _clear_all(self, *find_args):
+        self.store.find(*find_args).remove()
+
+    def clear_lanes(self):
+        self._clear_all(models_roadmap.Lane)
+
+    def clear_cards(self):
+        self._clear_all(models_roadmap.Card)
+
+    def store_lane(self, lane):
+        self.store.add(lane)
+
+    def store_card(self, card):
+        self.store.add(card)
+
+    def clear_todays_blueprint_daily_count_per_state(self):
+        self._clear_all(
+            models_roadmap.BlueprintDailyCountPerState,
+            models_roadmap.BlueprintDailyCountPerState.day == datetime.date.today())
+
+    def store_roadmap_bp_count_per_state(self):
+        query = """
+            SELECT implementation, lane_id, count(*)
+            FROM specs
+            JOIN meta on spec = specs.name
+            JOIN card on roadmap_id = value
+            WHERE key = 'Roadmap id'
+            GROUP BY implementation, lane_id
+            """
+        day = datetime.date.today()
+        result = self.store.execute(query)
+        for status, lane_id, count in result:
+            obj = models_roadmap.BlueprintDailyCountPerState()
+            obj.day = day
+            obj.status = status
+            obj.lane_id = lane_id
+            obj.count = count
+            self.store.add(obj)
+
+    def lane_is_collected(self, lane_id):
+        return self.store.find(models_roadmap.Lane, models_roadmap.
+                               Lane.lane_id == lane_id).one() is not None
+
+
+def get_json_item(data, item_name):
+    item = data[item_name]
+    if item is not None:
+        item = item.strip()
+    return unicode_or_None(item)
+
+
+def lookup_kanban_priority(numeric_priority):
+    priority_lookup = {-1: "low",
+                        0: "normal",
+                        1: "high"}
+    assert numeric_priority in priority_lookup, (
+        "Priority '%s' is unknown." % numeric_priority)
+    return unicode_or_None(priority_lookup[numeric_priority])

=== modified file 'lpworkitems/database.py'
--- lpworkitems/database.py	2011-06-24 19:09:26 +0000
+++ lpworkitems/database.py	2012-10-09 09:20:30 +0000
@@ -7,13 +7,13 @@
     store.execute('''CREATE TABLE version (
         db_layout_ref INT NOT NULL
         )''')
-    store.execute('''INSERT INTO version VALUES (10)''')
+    store.execute('''INSERT INTO version VALUES (15)''')
 
     store.execute('''CREATE TABLE specs (
         name VARCHAR(255) PRIMARY KEY,
         url VARCHAR(1000) NOT NULL,
         priority CHAR(20),
-        implementation CHAR(30),
+        implementation CHAR(30) NOT NULL,
         assignee CHAR(50),
         team CHAR(50),
         status VARCHAR(5000) NOT NULL,
@@ -25,6 +25,13 @@
         roadmap_notes VARCHAR(5000)
         )''')
 
+    store.execute('''CREATE TABLE spec_daily_count_per_state (
+        status VARCHAR(5000) NOT NULL,
+        day DATE NOT NULL,
+        lane_id REFERENCES lane(lane_id),
+        count INT NOT NULL
+        )''')
+
     store.execute('''CREATE TABLE work_items (
         description VARCHAR(1000) NOT NULL,
         spec VARCHAR(255) REFERENCES specs(name),
@@ -90,6 +97,30 @@
         display_name VARCHAR(50)
         )''')
 
+    store.execute('''CREATE TABLE lane (
+        name VARCHAR(200) NOT NULL,
+        lane_id NOT NULL,
+        is_current BOOLEAN,
+        cards REFERENCES card(card_id)
+        )''')
+
+    store.execute('''CREATE TABLE card (
+        name VARCHAR(200) NOT NULL,
+        card_id NOT NULL,
+        url VARCHAR(200),
+        is_healthy BOOLEAN,
+        status VARCHAR(50),
+        team VARCHAR(50),
+        priority VARCHAR(50),
+        size VARCHAR(50),
+        sponsor VARCHAR(50),
+        contact VARCHAR(50),
+        description BLOB,
+        acceptance_criteria BLOB,
+        roadmap_id VARCHAR(50),
+        lane_id REFERENCES lane(lane_id)
+        )''')
+
 
 def upgrade_if_needed(store):
     # upgrade DB layout
@@ -177,7 +208,47 @@
             )''')
         store.execute('UPDATE version SET db_layout_ref = 10')
         ver = 10
+    if ver == 10:
+        store.execute('''CREATE TABLE lane (
+            name VARCHAR(200) NOT NULL,
+            lane_id NOT NULL,
+            cards REFERENCES card(card_id)
+            )''')
 
+        store.execute('''CREATE TABLE card (
+            name VARCHAR(200) NOT NULL,
+            card_id NOT NULL,
+            status VARCHAR(50),
+            lane_id REFERENCES lane(lane_id)
+            )''')
+        store.execute('UPDATE version SET db_layout_ref = 11')
+        ver = 11
+    if ver == 11:
+        store.execute('ALTER TABLE card ADD COLUMN roadmap_id VARCHAR(50)')
+        store.execute('UPDATE version SET db_layout_ref = 12')
+        ver = 12
+    if ver == 12:
+        store.execute('ALTER TABLE card ADD COLUMN team VARCHAR(50)')
+        store.execute('ALTER TABLE card ADD COLUMN priority VARCHAR(50)')
+        store.execute('ALTER TABLE card ADD COLUMN size VARCHAR(50)')
+        store.execute('ALTER TABLE card ADD COLUMN sponsor VARCHAR(50)')
+        store.execute('ALTER TABLE card ADD COLUMN contact VARCHAR(50)')
+        store.execute('ALTER TABLE card ADD COLUMN description BLOB')
+        store.execute('ALTER TABLE card ADD COLUMN acceptance_criteria BLOB')
+        store.execute('ALTER TABLE lane ADD COLUMN is_current BOOLEAN')
+        store.execute('UPDATE version SET db_layout_ref = 13')
+    if ver == 13:
+        store.execute('ALTER TABLE card ADD COLUMN url VARCHAR(200)')
+        store.execute('ALTER TABLE card ADD COLUMN is_healthy BOOLEAN')
+        store.execute('UPDATE version SET db_layout_ref = 14')
+    if ver == 14:
+        store.execute('''CREATE TABLE spec_daily_count_per_state (
+            status VARCHAR(5000) NOT NULL,
+            day DATE NOT NULL,
+            lane_id REFERENCES lane(lane_id),
+            count INT NOT NULL
+            )''')
+        store.execute('UPDATE version SET db_layout_ref = 15')
 
 def get_store(dbpath):
     '''Open/initialize database.
@@ -205,5 +276,6 @@
     store.execute('''CREATE INDEX work_items_date_idx ON work_items (date)''')
     store.execute('''CREATE INDEX work_items_status_idx ON work_items (status)''')
 
+
 def create_v6_indexes(store):
     store.execute('''CREATE INDEX work_items_assignee_milestone_idx on work_items(assignee,milestone)''')

=== modified file 'lpworkitems/error_collector.py'
--- lpworkitems/error_collector.py	2011-06-08 19:30:24 +0000
+++ lpworkitems/error_collector.py	2012-10-09 09:20:30 +0000
@@ -48,6 +48,10 @@
         """Get the name of the blueprint, or None if not a blueprint."""
         return None
 
+    def get_project_name(self):
+        """Get the name of the project, or None if not a project."""
+        return None
+
     def format_for_display(self):
         """Produce a string representation of the Error.
 
@@ -84,6 +88,9 @@
     def get_blueprint_name(self):
         return self.blueprint.name
 
+    def get_project_name(self):
+        return self.blueprint.url.split('/')[-3]
+
 
 class BlueprintURLError(Error):
     """A deprecated class for backwards-compatibility.
@@ -101,6 +108,9 @@
     def get_blueprint_name(self):
         return self.blueprint_url.split('/')[-1]
 
+    def get_project_name(self):
+        return self.blueprint_url.split('/')[-3]
+
 
 class MilestoneError(Error):
 

=== modified file 'lpworkitems/factory.py'
--- lpworkitems/factory.py	2011-06-14 22:00:21 +0000
+++ lpworkitems/factory.py	2012-10-09 09:20:30 +0000
@@ -11,6 +11,7 @@
     TeamStructure,
     Workitem,
     )
+from lpworkitems.models_roadmap import BlueprintDailyCountPerState, Card
 
 
 class Factory(object):
@@ -63,6 +64,8 @@
             url = self.getUniqueUnicode(prefix=name+"_url")
         if status is None:
             status = self.getUniqueUnicode(prefix=name+"_status")
+        if implementation is None:
+            implementation = u'Unknown'
         blueprint.name = name
         blueprint.url = url
         blueprint.status = status
@@ -109,8 +112,11 @@
             self.store.add(workitem)
         return workitem
 
-    def make_meta(self, store=True):
+    def make_meta(self, key=None, value=None, blueprint=None, store=True):
         meta = Meta()
+        meta.key = key
+        meta.value = value
+        meta.blueprint = blueprint
         if store:
             self.store.add(meta)
         return meta
@@ -155,3 +161,28 @@
         if store:
             self.store.add(person)
         return person
+
+    def make_blueprint_daily_count_per_state(self, status=None, count=1,
+                                             day=None, store=True):
+        if status is None:
+            status = self.getUniqueUnicode()
+        if day is None:
+            day = datetime.date.today()
+        obj = BlueprintDailyCountPerState()
+        obj.day = day
+        obj.status = status
+        obj.count = count
+        obj.lane_id = 1
+        if store:
+            self.store.add(obj)
+        return obj
+
+    def make_card(self, store=True):
+        name = self.getUniqueUnicode()
+        card_id = self.getUniqueInteger()
+        lane_id = self.getUniqueInteger()
+        roadmap_id = self.getUniqueUnicode()
+        card = Card(name, card_id, lane_id, roadmap_id)
+        if store:
+            self.store.add(card)
+        return card

=== modified file 'lpworkitems/models.py'
--- lpworkitems/models.py	2011-12-06 15:20:43 +0000
+++ lpworkitems/models.py	2012-10-09 09:20:30 +0000
@@ -1,15 +1,20 @@
 import datetime
 import re
-
-from storm.locals import Date, Reference, ReferenceSet, Unicode
-
-
-def unicode_or_None(attr):
-    if attr is None:
-        return attr
-    if isinstance(attr, unicode):
-        return attr
-    return attr.decode("utf-8")
+from utils import unicode_or_None
+
+from storm.locals import Date, Int, Reference, ReferenceSet, Unicode
+
+ROADMAP_STATUSES_MAP = {
+    u'Completed': [u'Implemented'],
+    u'Blocked': [u'Needs Infrastructure', u'Blocked', u'Deferred'],
+    u'In Progress': [u'Deployment', u'Needs Code Review',
+                     u'Beta Available', u'Good progress',
+                     u'Slow progress', u'Started'],
+    u'Planned': [u'Unknown', u'Not started', u'Informational']}
+
+ROADMAP_ORDERED_STATUSES = ['Completed', 'In Progress', 'Blocked', 'Planned']
+assert set(ROADMAP_ORDERED_STATUSES) == set(ROADMAP_STATUSES_MAP.keys()), (
+    'The roadmap statuses are incorrect: %s' % ROADMAP_ORDERED_STATUSES)
 
 
 def fill_blueprint_info_from_launchpad(model_bp, lp_bp):
@@ -55,6 +60,14 @@
     project = Unicode()
 
 
+def get_roadmap_status_for_bp_implementation_status(implementation):
+    for key in ROADMAP_STATUSES_MAP:
+        if implementation in ROADMAP_STATUSES_MAP[key]:
+            return key
+    # XXX: Is None the appropriate return value here?
+    return None
+
+
 class Blueprint(object):
 
     __storm_table__ = "specs"
@@ -88,6 +101,15 @@
             lp_bp.whiteboard, "Roadmap\s+Notes")
         return model_bp
 
+    @property
+    def roadmap_status(self):
+        return get_roadmap_status_for_bp_implementation_status(
+            self.implementation)
+
+
+def current_date():
+    return datetime.date.today()
+
 
 class Person(object):
 
@@ -115,10 +137,6 @@
     superteam_name = Unicode(name="team")
 
 
-def current_date():
-    return datetime.date.today()
-
-
 class Meta(object):
 
     __storm_table__ = "meta"

=== added file 'lpworkitems/models_roadmap.py'
--- lpworkitems/models_roadmap.py	1970-01-01 00:00:00 +0000
+++ lpworkitems/models_roadmap.py	2012-10-09 09:20:30 +0000
@@ -0,0 +1,60 @@
+import datetime
+import re
+from utils import unicode_or_None
+
+from storm.locals import Date, Reference, ReferenceSet, Unicode, Int, Bool
+
+from lpworkitems import models
+
+
+class Card(object):
+
+    __storm_table__ = "card"
+
+    name = Unicode()
+    status = Unicode()
+    card_id = Int(primary=True)
+    lane_id = Int()
+    roadmap_id = Unicode()
+    team = Unicode()
+    priority = Unicode()
+    size = Unicode()
+    sponsor = Unicode()
+    contact = Unicode()
+    description = Unicode()
+    acceptance_criteria = Unicode()
+    url = Unicode()
+    is_healthy = Bool()
+
+    def __init__(self, name, card_id, lane_id, roadmap_id):
+        self.lane_id = lane_id
+        self.card_id = card_id
+        self.name = name
+        self.roadmap_id = roadmap_id
+
+
+class Lane(object):
+
+    __storm_table__ = "lane"
+
+    name = Unicode()
+    lane_id = Int(primary=True)
+    is_current = Bool()
+    cards = ReferenceSet(lane_id, Card.lane_id)
+
+    def __init__(self, name, lane_id):
+        self.lane_id = lane_id
+        self.name = name
+
+
+def current_date():
+    return datetime.date.today()
+
+
+class BlueprintDailyCountPerState(object):
+    __storm_table__ = 'spec_daily_count_per_state'
+    __storm_primary__ = 'status', 'day'
+    day = Date(default_factory=current_date)
+    status = Unicode()
+    lane_id = Int()
+    count = Int()

=== modified file 'lpworkitems/tests/test_collect.py'
--- lpworkitems/tests/test_collect.py	2011-06-14 22:00:21 +0000
+++ lpworkitems/tests/test_collect.py	2012-10-09 09:20:30 +0000
@@ -184,7 +184,6 @@
             self.store.find(
                 Milestone, Milestone.name == name).one())
 
-
     def test_store_blueprint_stores_blueprint(self):
         blueprint = self.factory.make_blueprint(store=False)
         ret = self.collector.store_blueprint(blueprint)

=== added file 'lpworkitems/tests/test_collect_roadmap.py'
--- lpworkitems/tests/test_collect_roadmap.py	1970-01-01 00:00:00 +0000
+++ lpworkitems/tests/test_collect_roadmap.py	2012-10-09 09:20:30 +0000
@@ -0,0 +1,69 @@
+import datetime
+
+from lpworkitems.collect_roadmap import (
+    CollectorStore,
+    get_json_item,
+    )
+from lpworkitems.models_roadmap import BlueprintDailyCountPerState
+from lpworkitems.error_collector import (
+    ErrorCollector,
+    )
+from lpworkitems.testing import TestCaseWithFakeLaunchpad
+
+
+class CollectorTests(TestCaseWithFakeLaunchpad):
+
+    def setUp(self):
+        super(CollectorTests, self).setUp()
+        self.error_collector = ErrorCollector()
+        self.collector = CollectorStore(
+            self.store, self.lp, self.error_collector)
+
+    def assertClears(self, cls, fn):
+        self.assertTrue(self.store.find(cls).count() > 0)
+        fn()
+        self.assertEqual(0, self.store.find(cls).count())
+
+    def test_clear_todays_blueprint_daily_count_per_state(self):
+        self.factory.make_blueprint_daily_count_per_state(
+            day=datetime.date.today())
+        self.assertClears(
+            BlueprintDailyCountPerState,
+            self.collector.clear_todays_blueprint_daily_count_per_state)
+
+    def test_store_roadmap_bp_count_per_state(self):
+        bp = self.factory.make_blueprint()
+        card = self.factory.make_card()
+        meta = self.factory.make_meta(
+            key=u'Roadmap id', value=card.roadmap_id, blueprint=bp)
+        self.collector.store_roadmap_bp_count_per_state()
+        self.assertEqual(
+            1, self.store.find(BlueprintDailyCountPerState).count())
+        entry = self.store.find(BlueprintDailyCountPerState).one()
+        self.assertEqual(1, entry.count)
+        self.assertEqual(card.lane_id, entry.lane_id)
+        self.assertEqual(bp.implementation, entry.status)
+
+    # XXX Add tests for the roadmap classes.
+
+
+class RoadmapUtilsTests(TestCaseWithFakeLaunchpad):
+
+    def setUp(self):
+        super(RoadmapUtilsTests, self).setUp()
+        self.json_data = {"data": "Text",
+                          "whitespace": " Text  ",
+                          "none": None
+                          }
+
+    def test_get_json_data_unicode(self):
+        item = get_json_item(self.json_data, 'data')
+        self.assertEquals(item, u'Text')
+
+    def test_get_json_data_whitespace(self):
+        item = get_json_item(self.json_data, 'whitespace')
+        self.assertEquals(item, u'Text')
+
+    def test_get_json_data_none(self):
+        item = get_json_item(self.json_data, 'none')
+        self.assertEquals(item, None)

=== modified file 'lpworkitems/tests/test_factory.py'
--- lpworkitems/tests/test_factory.py	2011-06-04 18:48:23 +0000
+++ lpworkitems/tests/test_factory.py	2012-10-09 09:20:30 +0000
@@ -162,7 +162,7 @@
         implementation = u"Implemented"
         self.assert_with_and_without(
             self.factory.make_blueprint, "implementation", implementation,
-            Equals(None))
+            Equals("Unknown"))
 
     def test_uses_assignee_name(self):
         assignee_name = self.factory.getUniqueUnicode(

=== modified file 'lpworkitems/tests/test_models.py'
--- lpworkitems/tests/test_models.py	2011-12-06 15:20:43 +0000
+++ lpworkitems/tests/test_models.py	2012-10-09 09:20:30 +0000
@@ -6,8 +6,11 @@
     extract_last_path_segment_from_url,
     extract_user_name_from_url,
     get_whiteboard_section,
-    )
-from lpworkitems.testing import TestCaseWithFakeLaunchpad
+    ROADMAP_STATUSES_MAP,
+    )
+from lpworkitems.testing import (
+    TestCaseWithFakeLaunchpad,
+    )
 
 
 class GetWhiteboardSectionTests(TestCase):
@@ -42,6 +45,18 @@
 
 class BlueprintTests(TestCaseWithFakeLaunchpad):
 
+    def test_roadmap_status(self):
+        roadmap_status = "Completed"
+        bp_implementation = ROADMAP_STATUSES_MAP[roadmap_status][0]
+        bp_status = self.factory.make_blueprint(
+            implementation=bp_implementation)
+        self.assertEqual(roadmap_status, bp_status.roadmap_status)
+
+    def test_roadmap_status_unknown_status(self):
+        blueprint = self.factory.make_blueprint(
+            implementation=u"Not Expected")
+        self.assertEqual(None, blueprint.roadmap_status)
+
     def test_from_launchpad_sets_name(self):
         name = self.factory.getUniqueUnicode(prefix="lpblueprint")
         lp_bp = self.lp.make_blueprint(name=name)

=== modified file 'report_tools.py'
--- report_tools.py	2012-07-17 06:00:48 +0000
+++ report_tools.py	2012-10-09 09:20:30 +0000
@@ -3,12 +3,25 @@
 # Tools for generating reports
 
 import datetime
-import urllib, sys, os.path, re
+<<<<<<< TREE
+import urllib, sys, os.path, re
+=======
+import urllib, sys, os.path, re
+from storm.locals import create_database, Store
+>>>>>>> MERGE-SOURCE
 from subprocess import Popen
 from cgi import escape
 from lpworkitems import database
 import errno
 import fcntl
+from lpworkitems.models_roadmap import (
+    Lane,
+    Card,
+)
+from lpworkitems.models import (
+    Meta, ROADMAP_STATUSES_MAP,
+    get_roadmap_status_for_bp_implementation_status,    
+)
 
 valid_states = [u'todo', u'blocked', u'inprogress', u'done', u'postponed']
 state_labels = [u'Todo', u'Blocked', u'In Progress', u'Done', u'Postponed']
@@ -180,6 +193,52 @@
         fh.close()
 
 
+def roadmap_pages(my_path, database, basename, config, lane, root=None):
+    cfg = load_config(config)
+    fh = open(basename + '.html', 'w')
+    chart_path, _ = os.path.split(basename)
+    chart_name = os.path.join(chart_path, 'current_quarter.svg')
+    try:
+        args = [os.path.join(my_path, 'html-report'), '-d', database]
+        args += ['--report-type', 'roadmap_page']
+        args += ['--lane', lane.name]
+        if root:
+            args += ['--root', root]
+        if lane.is_current:
+            args += ['--chart', chart_name]
+        report_args(args, theme=get_theme(cfg))
+        proc = Popen(args, stdout=fh)
+        print basename + '.html'
+        proc.wait()
+    finally:
+        fh.close()
+
+    if lane.is_current:
+        args = [os.path.join(my_path, 'roadmap-bp-chart'), '-d', database,
+                '-o', chart_name]
+        args += ['--inverted']
+        proc = Popen(args)
+        print chart_name
+        proc.wait()
+
+
+def roadmap_cards(my_path, database, basename, config, card, root=None):
+    cfg = load_config(config)
+    fh = open(basename + '.html', 'w')
+    try:
+        args = [os.path.join(my_path, 'html-report'), '-d', database]
+        args += ['--report-type', 'roadmap_card']
+        args += ['--card', '%s' % card.card_id]
+        if root:
+            args += ['--root', root]
+        report_args(args, theme=get_theme(cfg))
+        proc = Popen(args, stdout=fh)
+        print basename + '.html'
+        proc.wait()
+    finally:
+        fh.close()
+
+
 def run_reports(my_path, database, basename, config, milestone=None, team=None,
         user=None, trend_starts=None, trend_override=None, burnup=False, root=None, date=None):
 
@@ -281,6 +340,34 @@
     return escape(html, True)
 
 
+def blueprints_over_time(store):
+    '''Calculate blueprint development over time for the current lane.
+
+    We do not need to care about teams or groups since this is intended for the
+    roadmap overview.
+
+    Return date -> state -> count mapping. states are
+    {planned,inprogress,completed,blocked}.
+    '''
+    data = {}
+    result = store.execute("""
+        SELECT status, day, count
+        FROM spec_daily_count_per_state
+        JOIN lane on lane.lane_id = spec_daily_count_per_state.lane_id
+        WHERE lane.is_current = 1
+        """)
+    for status, day, count in result:
+        roadmap_status = get_roadmap_status_for_bp_implementation_status(
+            status)
+        assert roadmap_status is not None
+        if day not in data:
+            data[day] = {}
+        if roadmap_status not in data[day]:
+            data[day][roadmap_status] = 0
+        data[day][roadmap_status] += count
+    return data
+
+
 def workitems_over_time(store, team=None, group=None, milestone_collection=None):
     '''Calculate work item development over time.
 
@@ -876,6 +963,80 @@
     return rv
 
 
+def lanes(store):
+    return store.find(Lane)
+
+
+def lane(store, name, id=None):
+    if id is None:
+        return store.find(Lane, Lane.name == unicode(name)).one()
+    else:
+        return store.find(Lane, Lane.lane_id == id).one()
+
+
+def current_lane(store):
+    return store.find(Lane, Lane.is_current).one()
+
+
+def lane_cards(store, lane):
+    return lane.cards
+
+
+def statuses(store, lane):
+    result = []
+    for status in store.find(Card.status,
+                             Card.lane_id == lane.lane_id).config(distinct=True):
+        result.append((status, store.find(Card,
+                                          Card.lane_id == lane.lane_id,
+                                          Card.status == status)))
+    return result
+
+
+def cards(store):
+    return store.find(Card)
+
+
+def card(store, card_id):
+    return store.find(Card, Card.card_id == card_id)
+
+
+def card_blueprints(store, roadmap_id):
+    metas = store.find(Meta,
+                       Meta.key == u'Roadmap id',
+                       Meta.value == roadmap_id)
+    return [meta.blueprint for meta in metas]
+
+
+def card_blueprints_by_status(store, roadmap_id):
+    blueprints = card_blueprints(store, roadmap_id)
+    bp_by_status = {}
+    for key in ROADMAP_STATUSES_MAP:
+        bp_by_status[key] = []
+    for bp in blueprints:
+        bp_by_status[bp.roadmap_status].append(bp)
+    return bp_by_status
+
+
+def card_bp_status_counts(store, roadmap_id):
+    blueprints = card_blueprints(store, roadmap_id)
+    total_by_status = dict([(key, 0) for key in ROADMAP_STATUSES_MAP])
+    for bp in blueprints:
+        total_by_status[bp.roadmap_status] += 1
+    return total_by_status
+
+
+def check_card_health(store, card_health_checks, card):
+    performed_checks = []
+    card.is_healthy = True
+    for check in card_health_checks:
+        result = check.execute(card, store)
+        if result == check.NOT_OK:
+            card.is_healthy = False
+        performed_checks.append({'name': check.name,
+                                 'result': result})
+    return performed_checks
+
+
 def subteams(store, team):
     result = store.execute('SELECT name from team_structure where team = ?', (unicode(team),))
     return [i[0] for i in result]

=== added file 'roadmap-bp-chart'
--- roadmap-bp-chart	1970-01-01 00:00:00 +0000
+++ roadmap-bp-chart	2012-10-09 09:20:30 +0000
@@ -0,0 +1,249 @@
+#!/usr/bin/python
+#
+# Create a blueprint tracking chart from a blueprint database.
+#
+# Copyright (C) 2010, 2011 Canonical Ltd.
+# License: GPL-3
+
+import optparse, datetime, sys
+import report_tools
+
+from pychart import *
+
+def date_to_ordinal(s):
+    '''Turn yyyy-mm-dd strings to ordinals'''
+    return report_tools.date_to_python(s).toordinal()
+
+
+def ordinal_to_date(ordinal):
+    '''Turn an ordinal date into a string'''
+    d = datetime.date.fromordinal(int(ordinal))
+    return d.strftime('%Y-%m-%d')
+
+def format_date(ordinal):
+    d = datetime.date.fromordinal(int(ordinal))
+    return '/a60{}' + d.strftime('%b %d, %y')
+
+def do_chart(data, start_date, end_date, trend_start, title, filename, only_weekdays, inverted):
+    #set up default values
+    format = 'svg'
+    height = 450
+    width = 1000
+    legend_x = 700
+    legend_y = 200
+    title_x = 300
+    title_y = 350
+
+    if inverted:
+        legend_x=200
+
+    # Tell pychart to use colors
+    theme.use_color = True
+    theme.default_font_size = 12
+    theme.reinitialize()
+
+    # turn into pychart data model and calculate maximum number of WIs
+    max_items = 1 # start at 1 to avoid zero div
+    lastactive = 0
+    pcdata = []
+
+    for date in xrange(date_to_ordinal(start_date), date_to_ordinal(end_date)+1):
+        if (not only_weekdays or datetime.date.fromordinal(date).weekday() < 5):
+            end_date = ordinal_to_date(date)
+            i = data.get(ordinal_to_date(date), {})
+            count = i.get('Completed', 0) + i.get('Planned', 0) + i.get('Blocked', 0) + i.get('In Progress', 0)
+            if max_items < count:
+                max_items = count
+            pcdata.append((date, i.get('Planned', 0),0,
+                           i.get('Blocked', 0),0,
+                           i.get('In Progress', 0),0,
+                           i.get('Completed',0),0, count))
+            if count > 0:
+                lastactive = len(pcdata) - 1
+
+    # add some extra space to look nicer
+    max_items = int(max_items * 1.05)
+
+    x_interval = len(pcdata)/20
+    if max_items > 500:
+        y_interval = max_items/200*10
+    elif max_items < 20:
+        y_interval = 1
+    else:
+        y_interval = max_items/20
+
+    # create the chart object
+    chart_object.set_defaults(area.T, size=(width, height),
+            y_range=(0, None), x_coord=category_coord.T(pcdata, 0))
+
+    # tell the chart object it will use a bar chart, and will
+    # use the data list for it's model
+    chart_object.set_defaults(bar_plot.T, data=pcdata)
+
+    # create the chart area
+    # tell it to start at coords 0,0
+    # tell it the labels, and the tics, etc..
+    # HACK: to prevent 0 div
+    if max_items == 0:
+        max_items = 1
+    ar = area.T(legend=legend.T(loc=(legend_x,legend_y)), loc=(0,0),
+            x_axis=axis.X(label='Date', tic_interval=x_interval,format=format_date),
+            y_axis=axis.Y(label='Blueprints', tic_interval=y_interval),
+            y_range=(0, max_items))
+
+    #initialize the blar_plot fill styles
+    bar_plot.fill_styles.reset()
+
+    # create each set of data to plot
+    # note that index zero is the label col
+    # for each column of data, tell it what to use for the legend and
+    # what color to make the bar, no lines, and
+    # what plot to stack on
+
+    tlabel = ''
+
+    if inverted:
+        plot1 = bar_plot.T(label='Completed' + tlabel, hcol=7)
+        plot1.fill_style = fill_style.Plain(bgcolor=color.seagreen)
+
+        plot3 = bar_plot.T(label='In Progress' + tlabel, hcol=5, stack_on = plot1)
+        plot3.fill_style = fill_style.Plain(bgcolor=color.gray65)
+
+        plot5 = bar_plot.T(label='Blocked' + tlabel, hcol=3, stack_on = plot3)
+        plot5.fill_style = fill_style.Plain(bgcolor=color.red1)
+
+        plot7 = bar_plot.T(label='Planned' + tlabel, hcol=1, stack_on = plot5)
+        plot7.fill_style = fill_style.Plain(bgcolor=color.darkorange1)
+    else:
+        plot1 = bar_plot.T(label='Planned' + tlabel, hcol=1)
+        plot1.fill_style = fill_style.Plain(bgcolor=color.darkorange1)
+
+        plot3 = bar_plot.T(label='Blocked' + tlabel, hcol=3, stack_on = plot1)
+        plot3.fill_style = fill_style.Plain(bgcolor=color.red1)
+
+        plot5 = bar_plot.T(label='In Progress' + tlabel, hcol=5, stack_on = plot3)
+        plot5.fill_style = fill_style.Plain(bgcolor=color.gray65)
+
+        plot7 = bar_plot.T(label='Completed' + tlabel, hcol=7, stack_on = plot5)
+        plot7.fill_style = fill_style.Plain(bgcolor=color.seagreen)
+
+
+    plot1.line_style = None
+    plot3.line_style = None
+    plot5.line_style = None
+    plot7.line_style = None
+
+    plot11 = bar_plot.T(label='total', hcol=9)
+    plot11.fill_style = None
+    plot11.line_style = line_style.gray30
+
+    # create the canvas with the specified filename and file format
+    can = canvas.init(filename,format)
+
+    # add the data to the area and  draw it
+    ar.add_plot(plot1, plot3, plot5, plot7)
+    ar.draw()
+
+    # title
+    tb = text_box.T(loc=(title_x, title_y), text=title, line_style=None)
+    tb.fill_style = None
+    tb.draw()
+
+#
+# main
+#
+
+# argv parsing
+optparser = optparse.OptionParser()
+optparser.add_option('-d', '--database',
+    help='Path to database', dest='database', metavar='PATH')
+optparser.add_option('-t', '--team',
+        help='Restrict report to a particular team', dest='team')
+optparser.add_option('-m', '--milestone',
+        help='Restrict report to a particular milestone', dest='milestone')
+optparser.add_option('-o', '--output',
+        help='Output file', dest='output')
+optparser.add_option('--trend-start', type='int',
+        help='Explicitly set start of trend line', dest='trendstart')
+optparser.add_option('-u', '--user', 
+        help='Run for this user', dest='user')
+optparser.add_option('--only-weekdays', action='store_true',
+        help='Skip Saturdays and Sundays in the resulting graph', dest='only_weekdays')
+optparser.add_option('--inverted', action='store_true',
+        help='Generate an inverted burndown chart', dest='inverted')
+optparser.add_option('-s', '--start-date', 
+        help='Explicitly set the start date of the burndown data', dest='start_date')
+optparser.add_option('-e', '--end-date', 
+        help='Explicitly set the end date of the burndown data', dest='end_date')
+optparser.add_option('--no-foreign', action='store_true', default=False,
+        help='Do not show foreign totals separate', dest='noforeign')
+optparser.add_option('--group',
+        help='Run for this group', dest='group')
+optparser.add_option('--date',
+        help='Run for this date', dest='date')
+
+(opts, args) = optparser.parse_args()
+if not opts.database:
+    optparser.error('No database given')
+if not opts.output:
+    optparser.error('No output file given')
+
+if opts.user and  opts.team:
+    optparser.error('team and user options are mutually exclusive')
+if opts.user and opts.group:
+    optparser.error('user and group options are mutually exclusive')
+if opts.team and opts.group:
+    optparser.error('team and group options are mutually exclusive')
+if opts.milestone and opts.date:
+    optparser.error('milestone and date options are mutually exclusive')
+
+# The typing allows polymorphic behavior
+if opts.user:
+    opts.team = report_tools.user_string(opts.user)
+elif opts.team:
+    opts.team = report_tools.team_string(opts.team)
+
+store = report_tools.get_store(opts.database)
+
+milestone_collection = None
+if opts.milestone:
+    milestone_collection = report_tools.get_milestone(store, opts.milestone)
+elif opts.date:
+    milestone_collection = report_tools.MilestoneGroup(
+        report_tools.date_to_python(opts.date))
+
+
+# get date -> state -> count mapping
+data = report_tools.blueprints_over_time(store)
+
+if len(data) == 0:
+    print 'WARNING: no blueprints, not generating chart (team: %s, group: %s, due date: %s)' % (
+        opts.team or 'all', opts.group or 'none', milestone_collection and milestone_collection.display_name or 'none')
+    sys.exit(0)
+
+# calculate start/end date if no dates are given
+if opts.start_date is None:
+    start_date = sorted(data.keys())[0]
+else:
+    start_date=opts.start_date
+
+if opts.end_date is None:
+    if milestone_collection is not None:
+        end_date = milestone_collection.due_date_str
+    else:
+        end_date=report_tools.milestone_due_date(store)
+else:
+    end_date=opts.end_date
+
+if not start_date or not end_date or date_to_ordinal(start_date) > date_to_ordinal(end_date):
+    print 'WARNING: empty date range, not generating chart (team: %s, group: %s, due date: %s)' % (
+        opts.team or 'all', opts.group or 'none', milestone_collection and milestone_collection.display_name or 'none')
+    sys.exit(0)
+
+# title
+title = '/20all quarters'
+
+if milestone_collection is not None:
+    title += ' (%s)' % milestone_collection.name
+
+do_chart(data, start_date, end_date, opts.trendstart, title, opts.output, opts.only_weekdays, opts.inverted)

=== added file 'roadmap_health.py'
--- roadmap_health.py	1970-01-01 00:00:00 +0000
+++ roadmap_health.py	2012-10-09 09:20:30 +0000
@@ -0,0 +1,102 @@
+from report_tools import (
+    card_blueprints,
+    card_blueprints_by_status,
+)
+
+card_health_checks = []
+
+
+def register_health_check(cls):
+    card_health_checks.append(cls)
+    return cls
+
+
+class HealthCheck(object):
+    NOT_APPLICABLE = 'n/a'
+    OK = 'OK'
+    NOT_OK = 'Not OK'
+    name = 'Base check, not to be used'
+
+    @classmethod
+    def applicable(cls, card, store=None):
+        raise NotImplementedError()
+
+    @classmethod
+    def check(cls, card, store=None):
+        raise NotImplementedError()
+
+    @classmethod
+    def execute(cls, card, store=None):
+        if cls.applicable(card, store):
+            if cls.check(card, store):
+                return cls.OK
+            else:
+                return cls.NOT_OK
+        else:
+            return cls.NOT_APPLICABLE
+
+
+@register_health_check
+class DescriptionHealthCheck(HealthCheck):
+    name = 'Has description'
+
+    @classmethod
+    def applicable(cls, card, store=None):
+        return True
+
+    @classmethod
+    def check(cls, card, store=None):
+        return card.description is not None
+
+
+@register_health_check
+class CriteriaHealthCheck(HealthCheck):
+    name = 'Has acceptance criteria'
+
+    @classmethod
+    def applicable(cls, card, store=None):
+        return True
+
+    @classmethod
+    def check(cls, card, store=None):
+        return card.acceptance_criteria is not None
+
+
+@register_health_check
+class BlueprintsHealthCheck(HealthCheck):
+    name = 'Has blueprints'
+
+    @classmethod
+    def applicable(cls, card, store):
+        return card.status == 'Ready'
+
+    @classmethod
+    def check(cls, card, store):
+        return len(card_blueprints(store, card.roadmap_id)) > 0
+
+
+@register_health_check
+class BlueprintsBlockedHealthCheck(HealthCheck):
+    name = 'Has no Blocked blueprints'
+
+    @classmethod
+    def applicable(cls, card, store):
+        return card.status != 'Ready'
+
+    @classmethod
+    def check(cls, card, store):
+        blueprints = card_blueprints_by_status(store, card.roadmap_id)
+        return len(blueprints['Blocked']) == 0
+
+
+@register_health_check
+class RoadmapIdHealthCheck(HealthCheck):
+    name = 'Has a roadmap id'
+
+    @classmethod
+    def applicable(cls, card, store=None):
+        return True
+
+    @classmethod
+    def check(cls, card, store=None):
+        return card.roadmap_id != ''

=== modified file 'templates/base.html'
--- templates/base.html	2011-06-02 15:00:45 +0000
+++ templates/base.html	2012-10-09 09:20:30 +0000
@@ -127,6 +127,19 @@
 
     });
   </script>
+  <script type="text/javascript">
+
+    var _gaq = _gaq || [];
+    _gaq.push(['_setAccount', 'UA-16756069-4']);
+    _gaq.push(['_trackPageview']);
+
+    (function() {
+        var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+        ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+        var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+    })();
+
+ </script>
 </head>
 
 ${next.body()}

=== modified file 'templates/body.html'
--- templates/body.html	2011-05-27 20:07:25 +0000
+++ templates/body.html	2012-10-09 09:20:30 +0000
@@ -12,7 +12,7 @@
 % if page_type == "overview":
    active
 % endif
-	    " title="Overview" id="overview_nav"><a href="${util.url('')}">Overview</a></li>
+	    " title="Roadmap" id="overview_nav"><a href="${root}../lane/">Roadmap</a></li>
 	    <li class="link
 % if page_type == "about":
    active
@@ -39,6 +39,9 @@
 <div id="content_pane">
 <div id="main_content">
 ${next.body()}
+
+<%namespace name="footer" file="footer.html"/>
+${footer.body()}
 </div>
 </div>
 </div>

=== added file 'templates/footer.html'
=== added file 'templates/roadmap_card.html'
--- templates/roadmap_card.html	1970-01-01 00:00:00 +0000
+++ templates/roadmap_card.html	2012-10-09 09:20:30 +0000
@@ -0,0 +1,71 @@
+<%inherit file="body.html"/>
+<%namespace name="util" file="util.html"/>
+<%namespace name="terminology" file="terminology.html"/>
+
+<%!
+import report_tools
+%>
+
+<%def name="title()">
+${card_title}
+</%def>
+
+<h1>${title()}</h1>
+<div style="float: right">
+<h3>Health check</h3>
+
+<table>${'<tr colspan="2"><td><font color="#FF0000"><b>Needs attention!</b></font></td></tr>' if not card.is_healthy else ''}
+% for result in health_checks:
+    <tr ${'bgcolor="#FFAAAA"' if result['result'] == 'Not OK' else 'bgcolor="#FFFFFF"'}>
+        <td>${result['name']}</td>
+        <td>${result['result']}</td>
+    </tr>
+% endfor
+</table>
+</div>
+<h2>${card.status} in <a href="../lane/${lane}.html">${lane}</a></h2>
+<p>
+<ul>
+    <li>Card ID: <a href="${card.url}">${card.roadmap_id}</a>
+    <li>Sponsor: ${card.sponsor}
+    <li>Contact: ${card.contact}
+    <li>Priority: ${card.priority}
+    <li>Size: ${card.size}
+    <li>Team: ${card.team}
+</ul>
+
+<div style="clear:both; text-align: center">Overall blueprint completion</div>
+% if card_has_blueprints:
+${util.roadmap_progress_bar(bp_status_totals)}
+% else:
+<center><i>Progress graph pending linked blueprints.</i></center>
+% endif
+
+<h3>Description</h3> ${card.description if card.description is not None else '<i>No description could be found.</i>'}
+<p><a href="${card.url}">Read the full description</a>.
+<h3>Acceptance criteria</h3> ${card.acceptance_criteria if card.acceptance_criteria is not None else '<i>No acceptance criteria could be found.</i>'}
+<p><a href="${card.url}">Read the full acceptance criteria</a>.
+<p>
+% if card_has_blueprints:
+<table>
+<thead>
+    <tr><th>Title</th>
+    <th>Assignee</th>
+    <th>Priority</th>
+    <th>Status</th>
+    <th>Expected milestone</th>
+    </tr>
+</thead>
+% for status in status_order:
+% for bp in sorted(blueprints[status], key=lambda bp: bp.milestone_name):
+    <tr><td><a href="${bp.url}">${bp.name}</a></td>
+    <td>${bp.assignee_name}</td>
+    <td>${bp.priority}</td>
+    <td>${status}
+    <td>${bp.milestone_name}</td>
+    </tr>
+% endfor
+% endfor
+</table>
+% endif
+

=== added file 'templates/roadmap_lane.html'
--- templates/roadmap_lane.html	1970-01-01 00:00:00 +0000
+++ templates/roadmap_lane.html	2012-10-09 09:20:30 +0000
@@ -0,0 +1,60 @@
+<%inherit file="body.html"/>
+<%namespace name="util" file="util.html"/>
+<%namespace name="terminology" file="terminology.html"/>
+
+<%!
+import report_tools
+%>
+
+<%def name="title()">
+Progress for ${lane_title}
+</%def>
+
+<p style="text-align: right; color: green; font-size: 13pt; float: right">
+Lane:&nbsp;<select name="laneselect" onchange="window.location=this.value;">
+% for lane in lanes:
+<option value="${lane.name}.html"${' selected="selected"' if lane.name == lane_title else ''}>${lane.name}${' (current)' if lane.is_current else ''}</option>
+% endfor
+</select>
+<h1>${title()}</h1>
+${util.roadmap_progress_bar(bp_status_totals)}
+<p>
+<table width="100%">
+<thead><tr><th>Card</th><th>Status</th><th>Team</th><th>Priority</th><th>Blueprints</th><th>Health</th></tr></thead>
+% for status in statuses:
+% for card_dict in status['cards']:
+    <tr>
+    <td>
+        <a href="../card/${card_dict['card'].roadmap_id if card_dict['card'].roadmap_id != '' else card_dict['card'].card_id}.html">${card_dict['card'].name}</a>
+    </td>
+    <td>${status['name']}</td>
+    <td>${card_dict['card'].team}</td><td align=right>${card_dict['card'].priority}</td>
+    <td>
+      <div class="roadmap_wrap" title="
+% for bp_status in status_order:
+${bp_status}: ${card_dict['bp_statuses'][bp_status]}&nbsp;
+% endfor
+">
+% for bp_status in status_order:
+	<div class="roadmap_value" style="width:${card_dict['bp_percentages'][bp_status]}%">
+          <div class="${bp_status.replace(' ', '')}">&nbsp;</div>
+        </div>
+% endfor
+      </div>
+    </td>
+    <td>
+        ${'<font color="#FF0000">Needs attention!</font>' if not card_dict['card'].is_healthy else '' | n}
+    </td>
+% endfor
+% endfor
+</table>
+
+% if chart_url != 'burndown.svg':
+<!-- The cli option defaults to burndown.svg! :( -->
+<div class="overview_graph">
+<h3>Blueprint progress</h3><p><a href="current_quarter.svg">(enlarge)</a></p>
+<object
+	height="500" width="833"
+       	data="current_quarter.svg" type="image/svg+xml">Blueprint progress</object>
+</div>
+% endif

=== modified file 'templates/util.html'
--- templates/util.html	2011-06-01 20:55:59 +0000
+++ templates/util.html	2012-10-09 09:20:30 +0000
@@ -23,6 +23,15 @@
     </div>
 </%def>
 
+<%def name="roadmap_progress_bar(item)">
+<div class="roadmap_wrap" title="${item['Completed']} blueprints complete of ${item['Total']}">
+  <div class="roadmap_value" style="width:${item['Percentage']}%">
+    <div class="Completed">&nbsp;</div>
+  </div>
+    <div class="roadmap_progress_text">${item['Percentage']} % complete of ${item['Total']}</div>
+</div>
+</%def>
+
 <%def name="url(end)">${root}${end}</%def>
 
 <%def name="burndown_chart(chart_url, large=False)">

=== modified file 'tests.py'
--- tests.py	2011-06-01 03:36:17 +0000
+++ tests.py	2012-10-09 09:20:30 +0000
@@ -168,6 +168,7 @@
     loader = TestLoader()
     suite = loader.loadTestsFromName(__name__)
     suite.addTests(loader.loadTestsFromName("lpworkitems.tests.test_collect"))
+    suite.addTests(loader.loadTestsFromName("lpworkitems.tests.test_collect_roadmap"))
     suite.addTests(loader.loadTestsFromName("lpworkitems.tests.test_error_collector"))
     suite.addTests(loader.loadTestsFromName("lpworkitems.tests.test_factory"))
     suite.addTests(loader.loadTestsFromName("lpworkitems.tests.test_fake_launchpad"))

=== added file 'themes/linaro/templates/footer.html'
--- themes/linaro/templates/footer.html	1970-01-01 00:00:00 +0000
+++ themes/linaro/templates/footer.html	2012-10-09 09:20:30 +0000
@@ -0,0 +1,10 @@
+<%!
+    import datetime
+%>
+<div id="footer">
+    Last updated: ${datetime.datetime.utcnow().strftime("%a %d %B %Y, %H:%M UTC")} |
+    <a href="https://bugs.launchpad.net/launchpad-work-items-tracker">Bugs</a> |
+    <a href="https://code.launchpad.net/~linaro-infrastructure/launchpad-work-items-tracker/linaro">Code</a> |
+    <a href="https://code.launchpad.net/~linaro-infrastructure/launchpad-work-items-tracker/linaro-config">Config</a> |
+    <a href="/update.log.txt">Update log</a> (<a href="/update.log.txt.1">yesterday</a>)
+</div>

=== added file 'utils.py'
--- utils.py	1970-01-01 00:00:00 +0000
+++ utils.py	2012-10-09 09:20:30 +0000
@@ -0,0 +1,6 @@
+def unicode_or_None(attr):
+    if attr is None:
+        return attr
+    if isinstance(attr, unicode):
+        return attr
+    return attr.decode("utf-8")