Merge lp:~clint-fewbar/launchpad-work-items-tracker/server-team-mods into lp:launchpad-work-items-tracker

[Clint Byrum]

Hello tracker hackers.. this branch adds these features:

* Adds indexes making report and chart generation 5 - 6 times faster
* generate-all produces a burndown and report for every single user in the u directory
* inprogress status is now recorded for teams marked as 'inprogress_teams' in config, which currently is only 'canonical-server'

=== modified file 'burndown-chart'
--- burndown-chart	2010-07-13 08:55:43 +0000
+++ burndown-chart	2010-07-29 22:19:40 +0000
@@ -43,10 +43,11 @@
 
     for date in xrange(date_to_ordinal(start_date), date_to_ordinal(end_date)+1):
         i = data.get(ordinal_to_date(date), {})
-        count = i.get('done', 0) + i.get('todo', 0) + i.get('postponed', 0)
+        count = i.get('done', 0) + i.get('todo', 0) + i.get('inprogress', 0) + i.get('postponed', 0)
         if max_items < count:
             max_items = count
         pcdata.append((date, i.get('todo_teamonly', 0), i.get('todo', 0),
+            i.get('inprogress_teamonly', 0), i.get('inprogress', 0),
             i.get('done_teamonly', 0), i.get('done', 0),
             i.get('postponed_teamonly', 0), i.get('postponed', 0), count))
 
@@ -90,23 +91,30 @@
     plot2.fill_style = fill_style.Plain(bgcolor=color.coral1)
     plot2.line_style = None
 
-    plot3 = bar_plot.T(label='done (team)', hcol=3, stack_on = plot2)
-    plot3.fill_style = fill_style.green
+    plot3 = bar_plot.T(label='inprogress (team)', hcol=3, stack_on = plot2)
+    plot3.fill_style = fill_style.Plain(bgcolor=color.orange2)
     plot3.line_style = None
-    plot4 = bar_plot.T(label='done (foreign)', hcol=4, stack_on = plot2)
-    plot4.fill_style = fill_style.Plain(bgcolor=color.olivedrab1)
+    plot4 = bar_plot.T(label='inprogress (foreign)', hcol=4, stack_on = plot2)
+    plot4.fill_style = fill_style.Plain(bgcolor=color.orange1)
     plot4.line_style = None
 
-    plot5 = bar_plot.T(label="postponed (team)", hcol=5, stack_on = plot4)
-    plot5.fill_style = fill_style.Plain(bgcolor=color.yellow2)
+    plot5 = bar_plot.T(label='done (team)', hcol=5, stack_on = plot4)
+    plot5.fill_style = fill_style.green
     plot5.line_style = None
-    plot6 = bar_plot.T(label="postponed (foreign)  ", hcol=6, stack_on = plot4)
-    plot6.fill_style = fill_style.Plain(bgcolor=color.yellow1)
+    plot6 = bar_plot.T(label='done (foreign)', hcol=6, stack_on = plot4)
+    plot6.fill_style = fill_style.Plain(bgcolor=color.olivedrab1)
     plot6.line_style = None
 
-    plot7 = bar_plot.T(label='total', hcol=7)
-    plot7.fill_style = None
-    plot7.line_style = line_style.gray30
+    plot7 = bar_plot.T(label="postponed (team)", hcol=7, stack_on = plot6)
+    plot7.fill_style = fill_style.Plain(bgcolor=color.yellow2)
+    plot7.line_style = None
+    plot8 = bar_plot.T(label="postponed (foreign)  ", hcol=8, stack_on = plot6)
+    plot8.fill_style = fill_style.Plain(bgcolor=color.yellow1)
+    plot8.line_style = None
+
+    plot9 = bar_plot.T(label='total', hcol=9)
+    plot9.fill_style = None
+    plot9.line_style = line_style.gray30
 
     # create the canvas with the specified filename and file format
     can = canvas.init(filename,format)

=== modified file 'collect'
--- collect	2010-07-22 12:08:04 +0000
+++ collect	2010-07-29 22:19:40 +0000
@@ -359,7 +359,7 @@
     return None
 
 def lp_import_blueprint_workitems(lp, db, bp_name, bp_url, contents, release,
-        allow_inprogress = False):
+        inprogress_teams):
     '''Collect work items from a Launchpad blueprint.
 
     This includes work items from the whiteboard as well as linked bugs.
@@ -386,6 +386,16 @@
     dbg('lp_import_blueprint_workitems(): processing %s (spec milestone: %s, spec assignee: %s, spec implementation: %s)' % (
         bp_name, spec_milestone, spec_assignee, spec_implementation))
 
+    cur.execute('SELECT team FROM teams WHERE name = ?', (spec_assignee,))
+    assignee_teams = [t[0] for t in cur]
+
+    allow_inprogress = False
+    for t in assignee_teams:
+        if t in inprogress_teams:
+            dbg('  %s allows in progress due to membership in team %s' % (spec_assignee, t))
+            allow_inprogress = True
+            break
+
     cur.execute('SELECT name FROM milestones')
     valid_milestones = [m[0] for m in cur]
 
@@ -537,7 +547,7 @@
         dbg('lp_import(): downloading %s from %s' % (bp, url))
         contents = urllib.urlopen(url).read().decode('UTF-8')
         if lp_import_blueprint(db, bp, url, contents):
-            lp_import_blueprint_workitems(lp, db, bp, url, contents, cfg.get('release'))
+            lp_import_blueprint_workitems(lp, db, bp, url, contents, cfg.get('release'), cfg.get('inprogress_teams'))
 
 ########################################################################
 #
@@ -690,7 +700,7 @@
         cur.execute('''CREATE TABLE version (
             db_layout_ref INT NOT NULL
             )''')
-        cur.execute('''INSERT INTO version VALUES (4)''')
+        cur.execute('''INSERT INTO version VALUES (5)''')
 
         cur.execute('''CREATE TABLE specs (
             name VARCHAR(255) PRIMARY KEY,
@@ -742,6 +752,8 @@
             date TIMESTAMP NOT NULL
             )''')            
 
+        create_v5_indexes(cur)
+
         db.commit()
     else:
         # upgrade DB layout
@@ -788,9 +800,20 @@
             dbg('DB upgrade finished')
             ver = 4
 
+        if ver == 4:
+            dbg('Upgrading DB to layout version 5')
+            create_v5_indexes(cur)
+            cur.execute('UPDATE version SET db_layout_ref = 5')
+            db.commit()
+            ver = 5
 
     return db
 
+def create_v5_indexes(cur):
+    cur.execute('''CREATE INDEX teams_name_idx on teams(name)''')
+    cur.execute('''CREATE INDEX work_items_date_idx ON work_items (date)''')
+    cur.execute('''CREATE INDEX work_items_status_idx ON work_items (status)''')
+
 ########################################################################
 #
 # Program operations and main

=== modified file 'config/maverick.cfg'
--- config/maverick.cfg	2010-07-20 15:32:46 +0000
+++ config/maverick.cfg	2010-07-29 22:19:40 +0000
@@ -75,3 +75,4 @@
 
 recursive_teams = ['arm-ubuntu']
 
+inprogress_teams = ['canonical-server']

=== modified file 'generate-all'
--- generate-all	2010-04-06 13:48:48 +0000
+++ generate-all	2010-07-29 22:19:40 +0000
@@ -42,9 +42,41 @@
 milestones = [i[0] for i in cur]
 cur.execute('SELECT DISTINCT team FROM teams')
 teams = [i[0] for i in cur]
+cur.execute('SELECT DISTINCT name FROM teams')
+users = [i[0] for i in cur]
 
 my_path = os.path.dirname(sys.argv[0])
 
+usersubdir = os.path.join(opts.output_dir, 'u')
+try:
+    os.mkdir(usersubdir)
+except OSError:
+    None
+
+for u in users:
+    for m in milestones:
+        # entire cycle report for user
+        target = u + '-' + m
+        basename = os.path.join(usersubdir, target)
+        print basename + '.html'
+        f = open(basename + '.html', 'w')
+        subprocess.call([os.path.join(my_path, 'html-report'), '-d', opts.database,
+                '-t', u, '-m', m, '--chart', '%s.svg' % target], stdout=f)
+        f.close()
+
+        print basename + '.json'
+        f = open(basename + '.json', 'w')
+        subprocess.call([os.path.join(my_path, 'json-report'), '-d', opts.database,
+                '-t', u, '-m', m, '-c', opts.config], stdout=f)
+        f.close()
+
+        print basename + '.svg'
+        argv = [os.path.join(my_path, 'burndown-chart'), '-d', opts.database, '-t',
+            u, '-m', m, '-o', basename + '.svg']
+        if u in trend_starts:
+            argv += ['--trend-start', str(trend_starts[t])]
+        subprocess.call(argv)
+
 # per team/milestone reports
 for t in teams:
     for m in milestones:
@@ -68,6 +100,7 @@
             argv += ['--trend-start', str(trend_starts[(t, m)])]
         subprocess.call(argv)
 
+
     # entire cycle report for team
     basename = os.path.join(opts.output_dir, t)
     print basename + '.html'

=== modified file 'html-report'
--- html-report	2010-07-08 07:37:26 +0000
+++ html-report	2010-07-29 22:19:40 +0000
@@ -27,7 +27,7 @@
 <p>(Click header to sort)</p>
 <table id="byspecification">
  <thead>
-  <tr><th>Specification</th><th>Complexity</th> <th>todo</th><th>postponed</th><th>done</th> <th>Completion</th> <th>Priority</th> <th>Status</th></tr>
+  <tr><th>Specification</th><th>Complexity</th> <th>todo</th><th>inprogress</th><th>postponed</th><th>done</th> <th>Completion</th> <th>Priority</th> <th>Status</th></tr>
  </thead>
 '''
     data = report_tools.blueprint_completion(db, team, milestone)
@@ -35,16 +35,16 @@
     completion = []
     for (bp, i) in data.iteritems():
         completion.append((bp,
-            int(float(i['postponed']+i['done'])/(i['todo']+i['done']+i['postponed'])*100 + 0.5)))
+            int(float(i['postponed']+i['done'])/(i['todo']+i['done']+i['postponed']+i['inprogress'])*100 + 0.5)))
 
     completion.sort(key=lambda k: k[1]*100+report_tools.priority_value(data[k[0]]['priority']), reverse=True)
 
     for (bp, percent) in completion:
-        print '  <tr><td>%s</td> <td>%s</td><td>%i</td><td>%i</td><td>%i</td> <td>%i%%<br/>' \
+        print '  <tr><td>%s</td> <td>%s</td><td>%i</td><td>%i</td><td>%i</td><td>%i</td> <td>%i%%<br/>' \
             '<span style="font-size: 70%%">%s</span></td> <td>%s</td> <td>%s</td></tr>' % (
             '<a href="%s">%s</a>' % (data[bp]['url'], escape_html(bp)),
                 data[bp]['complexity'] or '',
-                data[bp]['todo'], data[bp]['postponed'], data[bp]['done'],
+                data[bp]['todo'], data[bp]['inprogress'], data[bp]['postponed'], data[bp]['done'],
                 percent,
                 data[bp]['implementation'],
                 format_priority(data[bp]['priority']),
@@ -58,24 +58,24 @@
 <p>(Click header to sort)</p>
 <table id="byassignee">
  <thead>
-  <tr><th>Assignee</th> <th>Complexity</th><th>todo</th><th>postponed</th><th>done</th>  <th>Completion</th></tr>
+  <tr><th>Assignee</th> <th>Complexity</th><th>todo</th><th>inprogress</th><th>postponed</th><th>done</th>  <th>Completion</th></tr>
  </thead>
 '''
     data = report_tools.assignee_completion(db, team, milestone)
 
     completion = []
     for a, i in data.iteritems():
-        (todo, postponed, done) = (len(i['todo']), len(i['postponed']), len(i['done']))
+        (todo, inprogress, postponed, done) = (len(i['todo']), len(i['inprogress']), len(i['postponed']), len(i['done']))
         completion.append((a,
-            int(float(postponed+done) / (todo+done+postponed)*100 + 0.5)))
+            int(float(postponed+done) / (todo+inprogress+done+postponed)*100 + 0.5)))
 
     completion.sort(key=lambda k: k[0], reverse=False)
 
     for (a, percent) in completion:
         a_html = escape_url(a or team or 'nobody')
         url = '%s/~%s/+specs?role=assignee' % (report_tools.blueprints_base_url, a_html)
-        print '  <tr><td><a href="%s">%s</a></td> <td>%s</td><td>%i</td><td>%i</td><td>%i</td> <td>%i%%</td></tr>' % (
-                url, a_html, data[a]['complexity'] or '', len(data[a]['todo']), len(data[a]['postponed']),
+        print '  <tr><td><a href="%s">%s</a></td> <td>%s</td><td>%i</td><td>%i</td><td>%i</td><td>%i</td> <td>%i%%</td></tr>' % (
+                url, a_html, data[a]['complexity'] or '', len(data[a]['todo']), len(data[a]['inprogress']), len(data[a]['postponed']),
                 len(data[a]['done']), percent)
     print '</table>'
 
@@ -103,11 +103,13 @@
         todo_len = len(i['todo'])
         postponed_len = len(i['postponed'])
         done_len = len(i['done'])
+        inprogress_len = len(i['inprogress'])
         a_html = escape_url(a or team or 'nobody')
         a_url = '%s/~%s/+specs?role=assignee' % (report_tools.blueprints_base_url, a_html)
         rows = {'todo': '<td rowspan="%s">todo</td>' % todo_len,
                 'postponed': '<td rowspan="%s">postponed</td>' % postponed_len,
-                'done': '<td rowspan="%s">done</td>' % done_len}
+                'done': '<td rowspan="%s">done</td>' % done_len,
+                'inprogress': '<td rowspan="%s">inprogress</td>' % inprogress_len}
         printed_assignee = False
         for status in report_tools.valid_states:
             printed_status = False
@@ -116,7 +118,7 @@
                 print '  <tr>',
                 if not printed_assignee:
                     print '<td rowspan="%s"><a href="%s" name="%s">%s</a></td> ' % (
-                        todo_len+postponed_len+done_len, a_url, escape_url(a or team or 'nobody'),
+                        todo_len+postponed_len+done_len+inprogress_len, a_url, escape_url(a or team or 'nobody'),
                         a_html)
                     printed_assignee = True
                 if not printed_status:

=== modified file 'report_tools.py'
--- report_tools.py	2010-07-08 00:38:33 +0000
+++ report_tools.py	2010-07-29 22:19:40 +0000
@@ -6,7 +6,7 @@
 import sqlite3 as dbapi2
 from cgi import escape
 
-valid_states = ['todo', 'done', 'postponed']
+valid_states = ['todo', 'done', 'postponed', 'inprogress']
 blueprints_base_url = 'https://blueprints.launchpad.net'
 
 def escape_sql(text):
@@ -59,7 +59,7 @@
     given team.
 
     Return date -> state -> count mapping. states are
-    {todo,done,postponed}{,_teamonly}.
+    {todo,inprogress,done,postponed}{,_teamonly}.
     '''
     data = {}
     if milestone:
@@ -119,7 +119,7 @@
     '''Determine current blueprint completion.
 
     Return blueprint -> info mapping, with info being a map with these
-    keys: todo, done, postponed, status, priority, implementation, url.
+    keys: todo, inprogress, done, postponed, status, priority, implementation, url.
     '''
     data = {}
     if milestone == '':
@@ -155,7 +155,7 @@
                     (s, last_date))
 
         for (bp, num, status, priority, impl, url, roadmap_notes) in cur:
-            info = data.setdefault(bp, {'todo': 0, 'done': 0, 'postponed': 0})
+            info = data.setdefault(bp, {'todo': 0, 'done': 0, 'postponed': 0, 'inprogress': 0})
             info[s] = num
             info['status'] = status or ''
             info['priority'] = priority
@@ -175,7 +175,7 @@
     '''Determine current blueprint tasks completion.
 
     Return blueprint -> info mapping, with info being a map with these
-    keys: todo, done, postponed, status, priority, implementation, url, tasks.
+    keys: todo, inprogress, done, postponed, status, priority, implementation, url, tasks.
     '''
     default = blueprint_completion(db, team=team)
     data = {}
@@ -227,7 +227,7 @@
     '''Determine current by-assignee completion.
 
     Return assignee -> info mapping with info being a map with these
-    keys: todo, done, postponed. Each of those values is a list of [blueprint,
+    keys: todo, inprogress, done, postponed. Each of those values is a list of [blueprint,
     workitem, priority, spec_url].
     '''
     data = {}
@@ -261,7 +261,7 @@
                     'WHERE w.status = ? AND w.date = ? %s ' % ms_sql,
                     (s, last_date))
         for (a, bp, description, priority, url) in cur:
-            info = data.setdefault(a, {'todo': [], 'done': [], 'postponed': []})
+            info = data.setdefault(a, {'todo': [], 'done': [], 'postponed': [], 'inprogress': []})
             info[s].append([bp, description, priority, url])
 
             cur2 = db.cursor();
@@ -288,7 +288,12 @@
 
     ### limit by a team 'like'? (see config)
     if team:
-        team_sql = "AND name like '%%%s%%' " % config.get('teams')[team]
+        lookup = ''
+        try:
+            lookup = config.get('teams')[team]
+        except KeyError:
+            lookup = team
+        team_sql = "AND name like '%%%s%%' " % lookup
     else:
         team_sql = ""
 
@@ -345,7 +350,7 @@
         ### use the todo/done/postponed numbers from blueprint_completion
         csd = completion_dict.get( spec_row["name"], None )
         if csd:
-            for k in ("todo","done","postponed"):
+            for k in valid_states:
                 spec_dict["completion"][k] = csd[k]