Merge lp:~isoschiz/endroid/kermit-v2 into lp:endroid

[Phil Connell]

An additional set of features/bugfixes for kermit:

 - Bunch of GUI improvements
 - Add log file viewer via HTTP (with a really slow, non-streaming interface. Should be fixed)
 - Many bug fixes from later testing

[Phil Connell (pconnell)]

Again, planning to review just the core.

Would like to see the database changes merged with any markups from -v1.

(Happy to do this myself if given write perms on the branch).

[Phil Connell (pconnell)]

A few minor database things.

Not clear on whether the V1 branch needs to merge too (e.g. the usermanagement changes).

review: Needs Fixing

Unmerged revisions

91. By Martin Morrison on 2014-05-16

Fixed a bunch more bugs from Matt's testing, including making some core things more configurable, and removing dependency on the calling user's environment

90. By Martin Morrison on 2014-05-11

Make some small visual tweaks to the filter bar

89. By Martin Morrison on 2014-05-11

Make some small visual tweaks to the filter bar

88. By Martin Morrison on 2014-05-11

Minor tweaks following review from Matt

87. By Martin Morrison on 2014-05-11

Fix a bug with infinite query loop when a query doesn't return the latest entry. Add ability to restrict the filters by a single type.

86. By Martin Morrison on 2014-05-10

Fix links that weren't working; javascript error from angular; and add a watermark to the filter bar. Just cos.

85. By Martin Morrison on 2014-05-09

Replace the filter bar with a new, improved version.

84. By Martin Morrison on 2014-05-09

Add sftp access to log files (including new UI)

83. By Martin Morrison on 2014-05-09

Fix delivery of templates to the right directory

82. By Martin Morrison on 2014-05-08

Merge in changes to clean up log files

=== added file 'bin/kermit'
--- bin/kermit	1970-01-01 00:00:00 +0000
+++ bin/kermit	2014-09-01 20:21:07 +0000
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+import os
+import pwd
+import sys
+import json
+import urllib
+import urllib2
+
+from argparse import ArgumentParser
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("-c", "--commitdiff", required=True)
+    parser.add_argument("-t", "--target", required=True)
+    parser.add_argument("-b", "--buglist", required=True)
+    parser.add_argument("-u", "--username",
+                        default=pwd.getpwuid(os.getuid())[0])
+    parser.add_argument("-a", "--address",
+                        required="KERMIT_URL" not in os.environ,
+                        default=os.environ.get("KERMIT_URL", None))
+
+    args = parser.parse_args()
+
+    data = urllib.urlencode({
+      'username': args.username,
+      'commitdiff': args.commitdiff,
+      'target': args.target,
+      'buglist': args.buglist,
+      'commitmsg': '',
+    })
+
+    req = urllib2.Request(args.address + '/kermit/submit', data)
+    try:
+        response = urllib2.urlopen(req)
+    except urllib2.HTTPError as err:
+        data = json.loads(err.read())
+        sys.stderr.write("Failed to submit commit to queue: {}\n"
+                         .format(data['fail']))
+        sys.exit(1)
+    else:
+        data = json.loads(response.read())
+        sys.stdout.write("Submitted commit to queue as {}\n"
+                         .format(data['success']))
+
+if __name__ == "__main__":
+    main()

=== added file 'bin/kermit_queue_runner'
--- bin/kermit_queue_runner	1970-01-01 00:00:00 +0000
+++ bin/kermit_queue_runner	2014-09-01 20:21:07 +0000
@@ -0,0 +1,1123 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+import pwd
+import json
+import time
+import shlex
+import shutil
+import logging
+import os.path
+import cfgparser
+import traceback
+import collections
+
+from argparse import ArgumentParser
+from StringIO import StringIO
+
+from twisted.python import log
+from twisted.spread import pb
+from twisted.internet import reactor, protocol, defer, task, error
+from twisted.protocols import basic
+from twisted.web.client import FileBodyProducer, Agent
+from twisted.web.http_headers import Headers
+
+
+_CFG_DEFAULTS = {
+    'filevars': '',
+    'attempts': '1',
+    'attempt_interval': '10',
+    'persist': 'no',
+    'environment': '',
+    'dependencies': '',
+}
+
+"""
+Utilities and types.
+
+"""
+
+_StageSpec = collections.namedtuple("_StageSpec", ("deps", "done"))
+
+class AbortedError(Exception): pass
+
+
+def makedirs(dirs):
+    """Make directories, but swallow any exceptions."""
+    try:
+        os.makedirs(dirs)
+    except OSError:
+        pass
+
+
+def _makesockname(dirname):
+    "Make the socket name for the inter-qr comms."
+    return "\0kermit#{}#status".format(dirname)
+
+
+def _send_update(qr, cfg, key, status):
+    """
+    Send an update to the kermit server for the given key.
+
+    """
+    logging.debug("Updating {} (running: {})"
+                  .format(key, status.get('running', [])))
+    statedir = os.path.join(qr.dir, "logs", key, ".kermit")
+    with open(os.path.join(statedir, "progress"), "w") as file:
+        file.write(json.dumps(status))
+
+    if not cfg.has_option("DEFAULT", "kermit_baseurl"):
+        return defer.succeed(None)
+
+    d = Agent(reactor).request(
+        "POST",
+        cfg.get("DEFAULT", "kermit_baseurl") + "/kermit/update/{}".format(key),
+        Headers({'Content-Type': ['text/json']}),
+        FileBodyProducer(StringIO(json.dumps(status)))
+    )
+    class BodyReceiver(protocol.Protocol):
+        def __init__(self, response, finished):
+            self.finished = finished
+            self.response = response
+            self.data = []
+        def dataReceived(self, bytes):
+            self.data.append(bytes)
+        def connectionLost(self, reason):
+            d, self.finished = self.finished, None
+            d.callback((self.response, self.data))
+    def _handle(response):
+        finished = defer.Deferred()
+        response.deliverBody(BodyReceiver(response, finished))
+        return finished
+    d.addCallback(_handle)
+    return d
+
+
+"""
+Command output logging between processes.
+
+Classes:
+ - _LoggingProcess: base class for processes that log their output.
+ - _LocalLoggingProcess: logs output to local files (used in primary ws).
+ - _RemoteLoggingProcess: logs output over the LogfileProtocol (used in shared
+   workspaces).
+ - _RemoteLogfileProtocol: the remote (i.e. sending) side of the
+   LogfileProtocol.
+ - _RemoteLogfileFactory: ClientFactory subclass for opening remote logfiles.
+ - _RemoteLogfile: wrapper around the transport to look file-like.
+ - _LocalLogfileProtocol: the local (i.e. receiving) side of the
+   LogfileProtocol.
+ - _LocalLogfileFactory: server Factory for opening remote logfiles.
+
+"""
+
+class _LoggingProcess(protocol.ProcessProtocol):
+    """
+    Wraps processes so that their output is logged to files.
+
+    Maintains two files:
+     - errlog: contains just the stderr content
+     - alllog: contains all output (stdout and stderr interleaved)
+
+    Provides two deferreds:
+     - ready: should be fired by subclasses when the log files are ready. When
+       spawning, the actual spawn should be predicated on this deferred.
+     - deferred: fires when the process completes, with either None, or the
+       failure reason.
+    Wraps processes so that their output is logged into the logs directory.
+
+    Maintains two logs. The first contains just the stderr content, and is
+    suffixed with .err.log; the second contains both stdout and stderr and is
+    suffixed with .log.
+
+    Callers should use the .deferred field to receive the result.
+
+    """
+    def __init__(self, name, logdir, key):
+        self.name = name
+        self.logdir = logdir
+        self.alllog = None
+        self.errlog = None
+        self.ready = defer.Deferred()
+        self.deferred = defer.Deferred()
+    def outReceived(self, data):
+        self.alllog.write(data)
+    def errReceived(self, data):
+        self.alllog.write(data)
+        self.errlog.write(data)
+    def processEnded(self, reason):
+        self.alllog.close()
+        self.errlog.close()
+        self.deferred, d = None, self.deferred
+        logging.debug("Process {} complete: {}"
+              .format(self.name, reason.value.exitCode))
+        if reason.value.exitCode == 0:
+            d.callback(None)
+        else:
+            d.errback(reason)
+
+class _LocalLoggingProcess(_LoggingProcess):
+    """
+    Logs to local files in the log directory.
+
+    alllog is given a .log suffix; errlog is given a .err.log suffix. There
+    are no async events required to set up the logs, so this fires the
+    ready callback on initialisation.
+
+    """
+    def __init__(self, name, logdir, key):
+        _LoggingProcess.__init__(self, name, logdir, key)
+        makedirs(logdir)
+        self.alllog = open(os.path.join(logdir, name + ".log"), "w")
+        self.errlog = open(os.path.join(logdir, name + ".err.log"), "w")
+        self.ready.callback(None)
+
+class _RemoteLoggingProcess(_LoggingProcess):
+    """
+    Logs to a remote file using a _RemoteLogfileFactory, requesting two files.
+
+    The first requested file is the alllog; the second is errlog. The ready
+    deferred fires once the two files are open.
+
+    """
+    def __init__(self, name, logdir, key):
+        _LoggingProcess.__init__(self, name, logdir, key)
+        f = _RemoteLogfileFactory(self.add_log_file, self._use_local)
+        logsockpath = "\0kermit#{}#logs".format(key)
+        reactor.connectUNIX(logsockpath, f)
+        reactor.connectUNIX(logsockpath, f)
+
+    def _use_local(self, failure):
+        failure.trap(error.ConnectionRefusedError)
+        logging.warning("Couldn't log remotely. Logging locally instead")
+        if self.alllog is None:
+            self.alllog = open(os.path.join("logs", self.name + ".log"), "w")
+        elif self.errlog is None:
+            self.errlog = open(os.path.join("logs",
+                                            self.name + ".err.log"), "w")
+            d, self.ready = self.ready, None
+            d.callback(None)
+        else:
+            raise ValueError("Didn't expect another remote logfile")
+
+    def add_log_file(self, proto):
+        if self.alllog is None:
+            proto.sendLine(self.name + ".log")
+            self.alllog = _RemoteLogfile(proto.transport)
+        elif self.errlog is None:
+            proto.sendLine(self.name + ".err.log")
+            self.errlog = _RemoteLogfile(proto.transport)
+            d, self.ready = self.ready, None
+            d.callback(None)
+        else:
+            raise ValueError("Didn't expect another remote logfile")
+
+
+class _RemoteLogfileProtocol(basic.LineReceiver):
+    def connectionMade(self):
+        try:
+            self.factory.add_log_file(self)
+        except Exception as e:
+            self.factory.clientConnectionFailed(None, e)
+
+class _RemoteLogfileFactory(protocol.ClientFactory):
+    protocol = _RemoteLogfileProtocol
+    def __init__(self, callback, errback):
+        self.add_log_file = callback
+        self.errback = errback
+    def clientConnectionFailed(self, _, reason):
+        logging.debug("Failed to connect to the logfile: {}".format(reason))
+        # Two things:
+        # - Should check if self.deferred is None (as we open 2 files and might
+        #   fail on the first)
+        # - Should probably not fail, and instead just log locally
+        self.errback(reason)
+
+class _RemoteLogfile(object):
+    """
+    Wrapper around _RemoteLogfileProtocol transport to make it file-like.
+    """
+    def __init__(self, transport):
+        self._transport = transport
+    def write(self, data):
+        self._transport.write(data)
+    def close(self):
+        self._transport.loseConnection()
+
+class _LocalLogfileProtocol(basic.LineReceiver):
+    def lineReceived(self, line):
+        logging.debug("Opening a log file: {}".format(line))
+        self._file = open(os.path.join(self.factory.op._cfg.get("DEFAULT",
+                                                                "logdir"),
+                                       line.strip()), "w")
+        self.setRawMode()
+    def rawDataReceived(self, data):
+        self._file.write(data)
+    def connectionLost(self, reason):
+        self._file.close()
+
+class _LocalLogfileFactory(protocol.Factory):
+    protocol = _LocalLogfileProtocol
+    def __init__(self, op):
+        self.op = op
+
+
+"""
+Operation objects and helper classes.
+
+"""
+
+class _RemoteResults(pb.Referenceable):
+    """
+    Allows a shared operation to communicate back to the local operation that
+    spawned it.
+
+    """
+    def __init__(self, op, stages):
+        self._op = op
+        self._stages = set(stages)
+
+    def remote_running(self, stage):
+        logging.debug("Our partner tells us {} has started running"
+                      .format(stage))
+        self._op._status['running'].append(stage)
+        self._op._send_update()
+
+    def remote_abort(self, stage):
+        if stage in self._op._status['running']:
+            self._op._status['running'].remove(stage)
+        if stage in self._stages:
+            logging.debug("Our partner tells us {} has aborted".format(stage))
+            self._stages.remove(stage)
+            self._op._not_doing(stage)
+            self._op._send_update()
+
+    def remote_done(self, stage):
+        if stage in self._op._status['running']:
+            self._op._status['running'].remove(stage)
+        if stage in self._stages:
+            logging.debug("Our partner tells us {} has succeeded".format(stage))
+            self._stages.remove(stage)
+            self._op._done(stage)
+            self._op._send_update()
+
+    def remote_fail(self, stage, reason):
+        if stage in self._op._status['running']:
+            self._op._status['running'].remove(stage)
+        logging.debug("Our partner tells us {} failed: {}"
+                      .format(stage, reason))
+        if stage in self._stages:
+            self._stages.remove(stage)
+        self._op._fail(stage, reason)
+        self._op._send_update()
+
+
+class _OperationBase(object):
+    """Base class for operations."""
+    def __init__(self, qr, cfgfile, cfg, target, section, status, ws=None):
+        self._qr = qr
+        self._key = cfg.get("DEFAULT", "key")
+        self._cfgfile = cfgfile
+        self._cfg = cfg
+        self._target = target
+        self._section = section
+        self._status = status
+        self._ws = ws
+
+        self._status['started_at'] = 0
+        self._status['completed_at'] = 0
+        self._status['stages'] = []
+        self._status.setdefault('done', [])
+        self._status.setdefault('done_at', {})
+        self._status['failed'] = []
+        self._status['failed_at'] = {}
+        self._status['failed_cleanup'] = False
+        self._status['check_failed'] = False
+        self._status['skipping'] = []
+        self._status['running'] = []
+        self._status['logdir'] = cfg.get("DEFAULT", "logdir")
+
+        opts = dict(cfg.items("DEFAULT"))
+        opts.update(dict(cfg.items(target)))
+        if self._ws:
+            opts.update(dict(cfg.items(self._ws, category="workspace")))
+        self._status['message'] = opts.get("message", "").format(**opts)
+
+        self._opts = opts
+        self._active_proc = None
+
+        logging.info("New Operation created: {}, {}, {}".format(self.__class__, self._ws, self._opts))
+
+    def _send_update(self):
+        return None
+
+    def begin(self):
+        pass
+
+    def complete(self):
+        pass
+
+    @defer.inlineCallbacks
+    def _run_stage(self, name, suppress_fail=False, suppress_done=False):
+        logging.debug("Running stage {}".format(name))
+        attempts = self._cfg.geteval(name, "attempts", category="command")
+        interval = self._cfg.geteval(name, "attempt_interval",
+                                     category="command")
+        self._status['running'].append(name)
+        if not suppress_fail and not suppress_done:
+            yield self._running(name)
+        yield self._send_update()
+        try:
+            # Perform multiple attempts if requested
+            while True:
+                try:
+                    yield self._spawn_cmd(name)
+                except Exception:
+                    attempts -= 1
+                    if attempts <= 0:
+                        raise
+                    # Sleep a bit between attempts
+                    logging.debug("Waiting to retry stage...")
+                    yield task.deferLater(reactor, interval, lambda: None)
+                else:
+                    break
+            # Also need to track errors in saving the logs
+            yield self._save_logs(name)
+        except Exception as e:
+            logging.exception(e)
+            if not suppress_fail:
+                yield self._fail(name, e)
+            raise e
+        else:
+            if not suppress_done:
+                yield self._done(name)
+        finally:
+            self._status['running'].remove(name)
+            yield self._send_update()
+
+    def _make_env(self, name, opts):
+        env = {}
+        for sec, cat in (('DEFAULT', None), (self._ws, "workspace"),
+                         (name, "command")):
+            if sec is None:
+                continue
+            for key, val in [e.split(':', 1)
+                             for e in self._cfg.getlist(sec, "environment",
+                                                        category=cat,
+                                                        default=[])]:
+                env[key.strip()] = val.strip().format(**opts)
+        return env
+
+    def _spawn_cmd(self, name):
+        section = self._cfg.section(name, category="command")
+        p = self.logger(name, self._cfg.get("DEFAULT", "logdir"), self._key)
+        myopts = self._opts.copy()
+
+        myopts.update(dict(section.items()))
+
+        # Load any files we need into variables
+        for key, filename in [e.split(':')
+                              for e in section.getlist('filevars')]:
+            filename = filename.strip().format(**myopts)
+            with open(filename.strip(), 'r') as file:
+                myopts[key] = file.read()
+
+        env = self._make_env(name, myopts)
+
+        args = map(lambda s: s.format(**myopts),
+                   shlex.split(section.get("command")))
+
+        pwd = section.get("pwd").format(**myopts)
+
+        with open(os.path.join(self._qr.dir, "logs",
+                               'commands.log'), 'a') as file:
+            file.write(str(args) + '\n')
+
+        def _spawn_proc(_):
+            logging.debug("Spawning: {} in {}".format(args, pwd))
+            proc = reactor.spawnProcess(p, args[0], args,
+                                        env=env,
+                                        path=os.path.join(self._qr.dir, pwd),
+                                        usePTY=True)
+            # Only track the active process if it's one of the stages we
+            # actually want to be able to abort. We don't want to abort pre
+            # cmds as that will result in aborting a workspace.
+            if name in self._stages:
+                self._active_proc = proc
+        p.ready.addCallbacks(_spawn_proc, p.deferred.errback)
+        def _clean_proc(_):
+            self._active_proc = None
+            return _
+        p.deferred.addBoth(_clean_proc)
+        return p.deferred
+
+    def _load_ws_state(self, ws):
+        statefile = os.path.join(self._qr.dir, ".kermit", ws + ".ws")
+        makedirs(os.path.dirname(statefile))
+        state = {}
+        if os.path.exists(statefile):
+            with open(statefile) as file:
+                state = json.loads(file.read())
+        return state
+
+    def _save_ws_state(self, ws, state):
+        statefile = os.path.join(self._qr.dir, ".kermit", ws + ".ws")
+        makedirs(os.path.dirname(statefile))
+        with open(statefile, "w") as file:
+            file.write(json.dumps(state))
+
+    @defer.inlineCallbacks
+    def setup_workspace(self, ws):
+        state = self._load_ws_state(ws)
+        if not state.get("setup", False):
+            for name in self._section.getlist("setup_cmds"):
+                yield self._run_stage(name, suppress_done=True)
+            else:
+                state["setup"] = True
+            self._save_ws_state(ws, state)
+
+    @defer.inlineCallbacks
+    def abort_workspace(self, ws):
+        for name in self._section.getlist("abort_cmds"):
+            try:
+                yield self._run_stage(name, suppress_fail=True,
+                                      suppress_done=True)
+            except Exception as e:
+                logging.debug("Failed to abort workspace")
+                logging.exception(e)
+                self._status['failed_abort'] = True
+        self._save_ws_state(ws, {"setup": False})
+
+    @defer.inlineCallbacks
+    def run_pre_cmds(self, suppress_fail=False, suppress_done=False):
+        for name in self._section.getlist("pre_cmds"):
+            yield self._run_stage(name, suppress_fail=suppress_fail,
+                                  suppress_done=suppress_done)
+        else:
+            self._status['started_at'] = time.time()
+            yield self._send_update()
+
+    @defer.inlineCallbacks
+    def run_post_cmds(self):
+        for name in self._section.getlist("post_cmds"):
+            try:
+                yield self._run_stage(name, suppress_fail=True,
+                                      suppress_done=True)
+            except Exception:
+                logging.debug("Failed to run post cmds")
+                traceback.print_exc()
+                self._status['failed_cleanup'] = True
+
+
+class _LocalOperation(_OperationBase):
+    logger = _LocalLoggingProcess
+    def __init__(self, qr, cfgfile, cfg, target, section, status):
+        _OperationBase.__init__(self, qr, cfgfile, cfg, target, section, status)
+        self._logconn = None
+        self._calc_stages()
+
+    def _calc_stages(self):
+        section = self._cfg.section(self._target)
+        cmds = section.getlist("commands")
+        requested = self._cfg.getlist("DEFAULT", "stages")
+        workspaces = section.getlist("workspaces", default=[])
+
+        for ws in self._cfg.getlist("DEFAULT", "ws_disable", default=[]):
+            if ws in workspaces:
+                workspaces.remove(ws)
+
+        logging.info("Only considering workspaces: {}".format(workspaces))
+
+        self._status['stages'] = cmds
+
+        # Deferred for each stage (fired when the stage completes)
+        ds = {k: defer.Deferred() for k in cmds}
+        # Mapping of stage to list of deferred it depends on
+        deps = collections.defaultdict(list)
+        # Mapping of workspace to list of stage names that are run in it
+        byws = collections.defaultdict(list)
+        # All stages that are being run
+        allstages = set()
+
+        def add_stage(stage, ws=None):
+            """
+            Add a stage to the list of stages to run.
+
+            Always adds any dependent stages first.
+
+            """
+            section = self._cfg.section(stage, category="command")
+            if ws is None:
+                ws = section.get("workspace", default=None)
+            if ws not in workspaces:
+                # Although this command could run in a separate workspace,
+                # that workspace is not enabled for this target
+                ws = None
+
+            if stage not in byws[ws]:
+                depds = []
+                for dep in section.getlist("dependencies"):
+                    if dep not in cmds:
+                        logging.error("Unknown stage {}. Ignoring".format(dep))
+                        continue
+                    add_stage(dep, ws)
+                    depds.append(ds[dep])
+
+                byws[ws].append(stage)
+                deps[stage].extend(depds)
+                allstages.add(stage)
+
+                # Remove any results that might exist, since we're redoing it
+                self._status.get('done_at', {}).pop(stage, None)
+                try:
+                    self._status.get('done', []).remove(stage)
+                except ValueError:
+                    pass
+
+        lastd = None
+        for cmd in cmds:
+            if lastd is not None:
+                deps[cmd].append(lastd)
+            lastd = ds[cmd]
+            # If the step hasn't been requested, skip it always
+            if requested and cmd not in requested:
+                self._status['skipping'].append(cmd)
+                continue
+            section = self._cfg.section(cmd, category='command')
+            # If the step has previous completed, and results persist, skip it
+            if cmd in self._status.get('done', []):
+                if section.geteval("persist"):
+                    continue
+            # In all cases, steps might be readded if they are dependencies of
+            # a step that needs to be completed
+            add_stage(cmd)
+
+        # Now for every stage that isn't being run, trigger the deferred
+        for cmd in cmds:
+            if cmd not in allstages:
+              ds[cmd].callback(None)
+
+        self._stages = {
+            k: _StageSpec(defer.DeferredList(deps[k], fireOnOneErrback=True),
+                          ds[k])
+            for k in cmds
+        }
+
+        self._byws = byws
+        self._final_deferred = defer.DeferredList(list(ds.values()),
+                                                  fireOnOneErrback=True)
+
+    @defer.inlineCallbacks
+    def run_cmds(self):
+        for name in self._byws[None]:
+            # First wait for any dependencies to complete (in case they
+            # are running in separate, shared workspaces)
+            logging.debug("Waiting for dependencies of {}".format(name))
+            yield self._deps(name)
+            logging.debug("Starting to run {}".format(name))
+            yield self._run_stage(name)
+        else:
+            logging.debug("Waiting for the end of all stages")
+            yield self._final_deferred
+            self._status['completed_at'] = time.time()
+
+    def _running(self, name):
+        pass
+
+    def _done(self, name):
+        if name not in self._status['done']:
+            self._status['done'].append(name)
+            self._status['done_at'][name] = time.time()
+
+        if name in self._stages:
+            self._stages[name].done.callback(None)
+            del self._stages[name]
+
+    def _fail(self, name, reason):
+        if name not in self._status['failed']:
+            self._status['failed'].append(name)
+            self._status['failed_at'][name] = time.time()
+
+        if name in self._stages:
+            self._stages[name].done.errback(reason)
+            del self._stages[name]
+
+    def _not_doing(self, name):
+        if name in self._stages:
+            self._stages[name].done.errback(AbortedError())
+            del self._stages[name]
+
+    def _deps(self, name):
+        return self._stages[name].deps
+
+    @defer.inlineCallbacks
+    def handover(self):
+        self._logconn = reactor.listenUNIX("\0kermit#{}#logs".format(self._key),
+                                           _LocalLogfileFactory(self))
+        for ws in self._byws:
+            if ws is not None:
+                yield self._handover(ws)
+
+    @defer.inlineCallbacks
+    def _handover(self, ws):
+        section = self._cfg.section(ws, category="workspace")
+        user = section.get("username", pwd.getpwuid(os.getuid())[0])
+        dir = self._cfg.get(self._target, "dir")
+        logging.debug("Handing over to {} (user: {}): {}"
+                      .format(ws, user, self._byws[ws]))
+
+        factory = pb.PBClientFactory()
+        sockname = _makesockname(os.path.join(dir, user))
+        logging.debug("Connecting to {}".format(repr(sockname)))
+        reactor.connectUNIX(sockname, factory)
+        root = yield factory.getRootObject()
+        yield root.callRemote("handover", _RemoteResults(self, self._byws[ws]),
+                              self._cfgfile, ws, self._byws[ws])
+
+    def begin(self):
+        self._status['begin_at'] = time.time()
+        
+    def complete(self):
+        if self._logconn:
+            self._logconn.stopListening()
+            self._logconn = None
+
+    @defer.inlineCallbacks
+    def abort(self):
+        for ws in self._byws:
+            if ws is not None:
+                yield self._abort(ws)
+        if self._active_proc:
+            self._active_proc.signalProcess("TERM")
+        if self._logconn:
+            self._logconn.stopListening()
+            self._logconn = None
+
+    @defer.inlineCallbacks
+    def _abort(self, ws):
+        section = self._cfg.section(ws, category="workspace")
+        user = section.get("username", pwd.getpwuid(os.getuid())[0])
+        dir = self._cfg.get(self._target, "dir")
+        logging.debug("Aborting {} (user: {})"
+              .format(ws, user))
+
+        factory = pb.PBClientFactory()
+        reactor.connectUNIX(_makesockname(os.path.join(dir, user)), factory)
+        root = yield factory.getRootObject()
+        yield root.callRemote("abort", self._cfg.get("DEFAULT", "key"), ws)
+
+    def _send_update(self):
+        """
+        Send an update to the kermit server for the given key.
+
+        """
+        return _send_update(self._qr, self._cfg, self._key, self._status)
+
+    @defer.inlineCallbacks
+    def _save_logs(self, name):
+        logs = self._cfg.getlist(name, "savelogs", default=[],
+                                 category="command")
+        logging.debug("Saving logs: {}".format(logs))
+        for log in logs:
+            log = log.format(**self._opts)
+            shutil.copy(log, self._cfg.get("DEFAULT", "logdir"))
+        yield None
+
+
+class _SharedOperation(_OperationBase):
+    logger = _RemoteLoggingProcess
+    def __init__(self, qr, cfgfile, cfg, target, section, remote_op,
+                 stages, ws):
+        _OperationBase.__init__(self, qr, cfgfile, cfg, target, section, {},
+                                ws=ws)
+
+        self._remote_op = remote_op
+        self._stages = stages
+
+    @defer.inlineCallbacks
+    def _save_logs(self, name):
+        logs = self._cfg.getlist(name, "savelogs", default=[],
+                                 category="command")
+        for log in logs:
+            log = log.format(**self._opts)
+            d = defer.Deferred()
+
+            def _transfer_file(proto):
+                proto.sendLine(os.path.basename(log))
+                fp = basic.FileSender()
+                infile = open(log, "r")
+                transfer = fp.beginFileTransfer(infile, proto.transport)
+                transfer.addBoth(lambda _: infile.close())
+                transfer.chainDeferred(d)
+
+            f = _RemoteLogfileFactory(_transfer_file, d.errback)
+
+            logsockpath = "\0kermit#{}#logs".format(self._opts['key'])
+            logging.debug("Saving log {}".format(log))
+            reactor.connectUNIX(logsockpath, f)
+            yield d
+
+    @defer.inlineCallbacks
+    def run_cmds(self):
+        for name in self._stages:
+            logging.debug("Starting to run {}".format(name))
+            yield self._run_stage(name)
+
+    def _running(self, name):
+        return self._remote_op.callRemote("running", name)
+
+    def _done(self, name):
+        return self._remote_op.callRemote("done", name)
+
+    def _fail(self, name, reason):
+        return self._remote_op.callRemote("fail", name, reason)
+
+    @defer.inlineCallbacks
+    def abort(self):
+        for name in self._stages:
+            yield self._remote_op.callRemote("abort", name)
+        if self._active_proc and self._stages:
+            # Only do this if we are still running stages. If there are no
+            # stages, then we are in cleanup, and should continue doing that.
+            self._active_proc.signalProcess("TERM")
+
+
+class QueueRunner(pb.Root):
+    """
+    Maintains a queue of config files to handle, and processes them one by one.
+
+    The QueueRunner can operate in one of two modes:
+     - as a persistent server, which processes queued up commit requests one
+       at a time.
+     - as a client, handing over any commit requests to an already running
+       server (this mode is implemented via a classmethod).
+
+    Once the server completes its entire queue, it exits.
+
+    The server provides updates to the central kermit (running inside EnDroid)
+    whenever progress is made.
+
+    """
+
+    def __init__(self, dir):
+        self.dir = dir
+        self._sockpath = _makesockname(dir)
+        self._workqueues = collections.defaultdict(collections.OrderedDict)
+        self._aborts = []
+        self._jobs = []
+        self._running = set()
+        self.conn = None
+
+        os.chdir(dir)
+        makedirs("logs")
+        makedirs(".kermit")
+
+        observer = log.PythonLoggingObserver()
+        observer.start()
+
+        logging.basicConfig(filename=os.path.join(".kermit", "qr.log"),
+                            level=logging.DEBUG,
+                            format="%(asctime)s[%(levelname)-3.3s] %(message)s",
+                            datefmt="%Y/%m/%d %H:%M:%S")
+        logging.getLogger().addHandler(logging.StreamHandler())
+
+    def enqueue(self, cfgfile):
+        logging.debug("Enqueuing another commit: {}".format(cfgfile))
+        reactor.callWhenRunning(self._prepare, cfgfile)
+
+    def abort(self, key, ws=None):
+        self._aborts.append((key, ws))
+        reactor.callWhenRunning(self._do_aborts)
+
+    def _consider_shutdown(self):
+        logging.debug("Considering shutdown, with following running: {}"
+                      .format(self._running))
+        if not self._running:
+            self.shutdown()
+
+    def shutdown(self):
+        logging.info("Queue Runner shutting down")
+        d = self.conn.stopListening()
+        d.addBoth(lambda _: reactor.stop())
+
+    @defer.inlineCallbacks
+    def _prepare(self, cfgfile):
+        cfg = cfgparser.CfgParser(_CFG_DEFAULTS)
+        cfg.read(cfgfile)
+
+        target = cfg.get("DEFAULT", "target")
+        key = cfg.get("DEFAULT", "key")
+        status = {}
+        operation = None
+
+        try:
+            status = yield self._check_checksum(cfg)
+
+            operation = _LocalOperation(self, cfgfile, cfg, target,
+                                        cfg.section(target), status)
+
+            # Hand over any items that should be run in a shared queue
+            yield operation.handover()
+        except Exception as e:
+            logging.error("Failed to prepare job {}".format(key))
+            logging.exception(e)
+
+            status['check_failed'] = str(e)
+
+            if operation is not None:
+                operation.complete()
+            
+            try:
+                yield _send_update(self, cfg, key, status)
+            except Exception as f:
+                logging.error("Failed to tell kermit we failed to prepare")
+                logging.exception(f)
+        else:
+            try:
+                yield operation._send_update()
+            except Exception as e:
+                logging.warning("Failed to update kermit: {}".format(str(e)))
+ 
+            self._workqueues[None][key] = operation
+        finally:
+            reactor.callWhenRunning(self._process_ws_queue, None)
+
+    @defer.inlineCallbacks
+    def _process_ws_queue(self, ws):
+        if ws in self._running:
+            return
+        logging.debug("Processing one item from queue: {}"
+                      .format(self._workqueues[ws]))
+        if not self._workqueues[ws]:
+            self._running.discard(ws)
+            logging.debug("No work for {}. Will consider shutting down shortly"
+                          .format(ws))
+            reactor.callLater(60, self._consider_shutdown)
+            return
+        self._running.add(ws)
+
+        key = iter(self._workqueues[ws]).next()
+        operation = self._workqueues[ws][key]
+
+        operation.begin()
+
+        # Quite a simple function really:
+        # - Run the pre_cmds
+        # - Run each of the main commands, after waiting for its deps to run
+        # - No matter what, run the post commands
+        # - Then finally, schedule another run of the function
+
+        try:
+            if ws in operation._cfg.getlist("DEFAULT", "ws_reset", default=[]):
+                logging.info("Resetting workspace as requested")
+                yield operation.abort_workspace(ws)
+                yield operation.setup_workspace(ws)
+
+            # Run the "check" commands first to ensure everything is ready
+            try:
+                logging.debug("Preparing existing workspace {}".format(ws))
+                yield operation.run_pre_cmds(suppress_fail=bool(ws),
+                                             suppress_done=bool(ws))
+            except Exception:
+                if ws:
+                    logging.debug("Exception occurred; resetting workspace {}"
+                                  .format(ws))
+                    yield operation.abort_workspace(ws)
+                    logging.debug("Trying to setup workspace {}".format(ws))
+                    yield operation.setup_workspace(ws)
+                else:
+                    raise
+
+            # This is the main command loop
+            yield operation.run_cmds()
+        except Exception as f:
+            # We'll already have notified kermit of the failure
+            logging.exception(f)
+            # But notify guys helping us to not bother anymore, or let the
+            # main ws know that the rest of the steps have been aborted.
+            try:
+                yield operation.abort()
+            except Exception as e:
+                logging.error("Failed to abort")
+                logging.exception(e)
+                # But just swallow the error now
+
+        finally:
+            # Run the post commands no matter what
+            yield operation.run_post_cmds()
+            operation.complete()
+            self._workqueues[ws].pop(key, None)
+
+            # Make sure to try to process another item from the queue
+            self._running.discard(ws)
+            reactor.callWhenRunning(self._process_ws_queue, ws)
+
+    def _do_aborts(self):
+        if not self.conn:
+            return
+        logging.debug("Aborting: {}".format(self._aborts))
+        logging.debug("Queues: {}".format(self._workqueues))
+        for key, ws in self._aborts:
+            logging.warning("Aborting {} in {}".format(key, ws))
+            if key in self._workqueues.get(ws, []):
+                self._workqueues[ws][key].abort()
+                del self._workqueues[ws][key]
+        del self._aborts[:]
+
+    def _check_checksum(self, cfg):
+        """
+        Load existing data for a commit (if any) after checking the checksum.
+
+        The checksum for the 'commitdiff' specified in the given configuration
+        is checked against any stored checksum from a previous run of the
+        same commit.
+
+        This function returns a deferred that will fire with the existing
+        status information, or an empty dict if none exists or the checksums
+        don't match.
+
+        """
+        statedir = os.path.join(self.dir, "logs",
+                                cfg.get("DEFAULT", "key"), ".kermit")
+        makedirs(statedir)
+
+        oldchecksum, progress = None, None
+        if os.path.exists(os.path.join(statedir, "checksum")):
+            with open(os.path.join(statedir, "checksum")) as file:
+                oldchecksum = file.read().strip()
+        if os.path.exists(os.path.join(statedir, "progress")):
+            with open(os.path.join(statedir, "progress")) as file:
+                progress = json.loads(file.read())
+
+        if not os.path.exists(cfg.get("DEFAULT", "commitdiff")):
+            raise ValueError("Specified commit diff doesn't exist or not "
+                             "readable.")
+
+        difffile = os.path.join(statedir, "commit.diff")
+        shutil.copy(cfg.get("DEFAULT", "commitdiff"), difffile)
+        cfg.set("DEFAULT", "commitdiff", difffile)
+
+        p = ProcessOutput()
+
+        def handleChecksum(newchecksum):
+            newchecksum = newchecksum.strip()
+            logging.debug("Checksum calculated as {} (was {}). Progress is {}"
+                  .format(newchecksum, oldchecksum, progress))
+            if newchecksum == oldchecksum:
+                return progress
+            else:
+                with open(os.path.join(statedir, "progress"), "w") as file:
+                    file.write(json.dumps({}))
+                with open(os.path.join(statedir, "checksum"), "w") as file:
+                    file.write(newchecksum)
+                return {}
+
+        p.deferred.addCallbacks(handleChecksum, lambda f: {})
+        reactor.spawnProcess(p, "md5sum", ("md5sum", difffile),
+                             env=os.environ, path=self.dir)
+        return p.deferred
+
+    def start_listening(self):
+        """
+        This (perhaps poorly named) method might change the qr into a client
+        if it fails to set up the server.
+
+        """
+        self.conn = reactor.listenUNIX(self._sockpath, pb.PBServerFactory(self))
+        logging.info("Kermit Queue Runner started")
+        reactor.callLater(60, self._consider_shutdown)
+
+    @staticmethod
+    def send_queues(dir, cfgfile=None, abort=None):
+        reactor.callWhenRunning(QueueRunner._send_queues, dir, cfgfile, abort)
+
+    @staticmethod
+    @defer.inlineCallbacks
+    def _send_queues(dir, cfgfile=None, abort=None):
+        # This means we failed to listen. So send instead.
+        logging.debug("Failed to listen, so sending our queue instead")
+        factory = pb.PBClientFactory()
+        reactor.connectUNIX(_makesockname(dir), factory)
+        root = yield factory.getRootObject()
+        try:
+            if cfgfile:
+                yield root.callRemote("enqueue", cfgfile)
+            if abort:
+                yield root.callRemote("abort", abort)
+        except Exception as e:
+            logging.debug("Total failure")
+            logging.debug(str(fail))
+        reactor.stop()
+
+    def remote_enqueue(self, cfgfile):
+        """Callback for enqueuing a regular commit (from a peer process)."""
+        self.enqueue(cfgfile)
+
+    def remote_abort(self, key, ws=None):
+        logging.debug("Being told to abort {} in {}".format(key, ws))
+        # Both local and remote workspaces can get this call!
+        self.abort(key, ws)
+
+    def remote_handover(self, remote_op, cfgfile, ws, stages):
+        logging.debug("Been handed something to do: {} in {} (stages: {})"
+                      .format(cfgfile, ws, stages))
+
+        cfg = cfgparser.CfgParser(_CFG_DEFAULTS)
+        cfg.read(cfgfile)
+
+        target = cfg.get("DEFAULT", "target")
+        key = cfg.get("DEFAULT", "key")
+
+        operation = _SharedOperation(self, cfgfile, cfg, target,
+                                     cfg.section(ws, category="workspace"),
+                                     remote_op, stages, ws=ws)
+
+        self._workqueues[ws][key] = operation
+        reactor.callWhenRunning(self._process_ws_queue, ws)
+
+
+class ProcessOutput(protocol.ProcessProtocol):
+    """
+    A nice simple ProcessProtocol that returns the stdout of the process.
+
+    Callers should use the .deferred field to receive the result.
+
+    """
+    def __init__(self):
+        self.deferred = defer.Deferred()
+        self.output = ""
+    def outReceived(self, data):
+        self.output += data
+    def processEnded(self, reason):
+        self.deferred, d = None, self.deferred
+        if reason.value.exitCode == 0:
+            d.callback(self.output)
+        else:
+            d.errback(reason)
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("-a", "--abort")
+    parser.add_argument("userdir")
+    parser.add_argument("cfgfile", nargs='?')
+
+    args = parser.parse_args()
+
+    qr = QueueRunner(args.userdir)
+    try:
+        qr.start_listening()
+    except error.CannotListenError as e:
+        QueueRunner.send_queues(args.userdir, args.cfgfile, args.abort)
+    else:
+        if args.cfgfile:
+            qr.enqueue(args.cfgfile)
+        if args.abort:
+            qr.abort(args.abort)
+
+    reactor.run()
+
+if __name__ == "__main__":
+    main()

=== added file 'bin/kermit_trigger'
--- bin/kermit_trigger	1970-01-01 00:00:00 +0000
+++ bin/kermit_trigger	2014-09-01 20:21:07 +0000
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import pwd
+import StringIO
+import argparse
+import subprocess
+import ConfigParser
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-w", "--workspace", action="store_true")
+    parser.add_argument("-a", "--abort")
+
+    args = parser.parse_args()
+
+    # Read the config file in from stdin
+    config = sys.stdin.read()
+
+    cfg = ConfigParser.SafeConfigParser({
+        'qr_path': 'kermit_queue_runner',
+    })
+    cfg.readfp(StringIO.StringIO(config))
+
+    target = cfg.get("DEFAULT", "target")
+
+    basedir = cfg.get(target, "dir")
+    userdir = os.path.join(basedir, pwd.getpwuid(os.getuid())[0])
+    if not os.path.exists(userdir):
+        os.makedirs(userdir)
+
+    cmd = ["screen", "-c", "/dev/null", "-d", "-m",
+           cfg.get("DEFAULT", "qr_path"), userdir]
+
+    key = cfg.get("DEFAULT", "key")
+    cfg.set("DEFAULT", "logdir", os.path.join(userdir, 'logs', key))
+
+    filename = os.path.join(userdir, ".kermit", key + ".cfg")
+    if not args.workspace and not args.abort:
+        if not os.path.exists(os.path.dirname(filename)):
+            os.makedirs(os.path.dirname(filename))
+
+        with open(filename, "w") as file:
+            cfg.write(file)
+
+        cmd.append(filename)
+
+    if args.abort:
+        cmd.extend(("-a", args.abort))
+
+    st = subprocess.call(cmd)
+    sys.exit(st)
+
+if __name__ == "__main__":
+    main()

=== modified file 'debian/control'
--- debian/control	2014-08-11 09:02:24 +0000
+++ debian/control	2014-09-01 20:21:07 +0000
@@ -12,3 +12,17 @@
 Description: XMPP bot framework
  EnDroid is a framework for building bots in XMPP. Its architecture
  is based around plugins, in order to make it as extensibile as possible.
+
+Package: endroid-kermit
+Architecture: all
+Depends: python,python-twisted,endroid,${misc:Depends}
+Suggests: endroid-kermit-daemons
+Description: Continuous Integration commit queue for EnDroid
+ Kermit is a CI commit queue handling plugin for EnDroid.
+
+Package: endroid-kermit-daemons
+Architecture: all
+Depends: python,python-twisted,screen,${misc:Depends}
+Description: Daemon processes for the Kermit EnDroid plugin
+ This package must be installed on any servers that will be used as Kermit
+ commit hosts.

=== added file 'debian/endroid-kermit-daemons.install'
--- debian/endroid-kermit-daemons.install	1970-01-01 00:00:00 +0000
+++ debian/endroid-kermit-daemons.install	2014-09-01 20:21:07 +0000
@@ -0,0 +1,3 @@
+bin/kermit usr/bin/
+bin/kermit_trigger usr/bin/
+bin/kermit_queue_runner usr/bin/

=== added file 'debian/endroid-kermit.install'
--- debian/endroid-kermit.install	1970-01-01 00:00:00 +0000
+++ debian/endroid-kermit.install	2014-09-01 20:21:07 +0000
@@ -0,0 +1,19 @@
+resources/kermit/profile.html usr/share/endroid/templates/kermit/
+resources/kermit/queue.html usr/share/endroid/templates/kermit/
+resources/kermit/response.json usr/share/endroid/templates/kermit/
+resources/kermit/alert.js usr/share/endroid/media/kermit/
+resources/kermit/queue.js usr/share/endroid/media/kermit/
+resources/kermit/profile.js usr/share/endroid/media/kermit/
+resources/kermit/services.js usr/share/endroid/media/kermit/
+resources/kermit/alert.css usr/share/endroid/media/kermit/
+resources/kermit/queue.css usr/share/endroid/media/kermit/
+resources/kermit/profile.css usr/share/endroid/media/kermit/
+resources/kermit/kermit.png usr/share/endroid/media/kermit/
+resources/kermit/kermit-logo.jpg usr/share/endroid/media/kermit/
+resources/kermit/kermit.css usr/share/endroid/media/kermit/
+resources/kermit/kermit-icon-120.png usr/share/endroid/media/kermit/
+resources/kermit/kermit-icon-152.png usr/share/endroid/media/kermit/
+resources/kermit/kermit-icon-60.png usr/share/endroid/media/kermit/
+resources/kermit/kermit-icon-76.png usr/share/endroid/media/kermit/
+resources/kermit/kermit-icon.png usr/share/endroid/media/kermit/
+resources/kermit/kermit_splash.png usr/share/endroid/media/kermit/

=== added file 'debian/endroid-kermit.pyinstall'
--- debian/endroid-kermit.pyinstall	1970-01-01 00:00:00 +0000
+++ debian/endroid-kermit.pyinstall	2014-09-01 20:21:07 +0000
@@ -0,0 +1,1 @@
+src/endroid/plugins/kermit/*py endroid.plugins.kermit

=== modified file 'debian/endroid.install'
--- debian/endroid.install	2014-04-05 19:53:27 +0000
+++ debian/endroid.install	2014-09-01 20:21:07 +0000
@@ -6,6 +6,6 @@
 bin/spelunk_hi5s usr/sbin/
 lib/wokkel-0.7.1-py2.7.egg usr/lib/endroid/dependencies
 var/endroid.db var/lib/endroid/db
-doc/EnDroid.png usr/share/endroid/media/
+resources/httpinterface/EnDroid.png usr/share/endroid/media/httpinterface/
 resources/httpinterface/index.html usr/share/endroid/templates/httpinterface/
 resources/httpinterface/notfound.html usr/share/endroid/templates/httpinterface/

=== added file 'debian/endroid.pyinstall'
--- debian/endroid.pyinstall	1970-01-01 00:00:00 +0000
+++ debian/endroid.pyinstall	2014-09-01 20:21:07 +0000
@@ -0,0 +1,3 @@
+src/endroid/*py endroid
+src/endroid/plugins/*py endroid.plugins
+src/endroid/plugins/compute/*py endroid.plugins.compute

=== added file 'etc/kermit.cfg'
--- etc/kermit.cfg	1970-01-01 00:00:00 +0000
+++ etc/kermit.cfg	2014-09-01 20:21:07 +0000
@@ -0,0 +1,157 @@
+# Configuration file for Kermit
+
+# Section names:
+#
+# - A naked section name describes a "target". This is something that the user
+#   types in, and provides the starting point for what configuration applies
+#   to a given commit.
+# - "workspace:" category sections describe auxilliary workspaces that may
+#   be used to perform commits. Note that these are in addition to the
+#   commit workspace that is created specifically for any given commit.
+# - "command:" category sections describe the commands that can be run to
+#   perform a commit.
+#
+# 'target' sections will reference 'workspace' and 'command' sections; and
+# 'workspace' sections can also reference 'command' sections; and 'command'
+# sections can also reference other 'command' sections.
+#
+# Within 'target' sections, the following items have meaning:
+#
+# - "dir" is the directory that all operations for this target are performed
+#   in. Note that multiple targets can share the same directory, in that the
+#   same queues simply get used per person. Each workspace (including the
+#   per-commit one) are rooted in this directory, so ensure commands won't
+#   conflict with other commands run in parallel.
+# - "workspaces" is a list of the auxilliary workspaces that may be used to
+#   perform commits to this target. If a given command requests a workspace
+#   that is not listed for that target, then the command is simply run in
+#   the commit workspace instead.
+# - "pre_cmds" and "post_cmds" are the lists of commands to run in the commit
+#   workspace before/after the commit commands themselves.
+# - "commands" is the list of commands that need to be run (and pass) to
+#   perform the actual commit. Each command implicitly depends upon the
+#   completion of the command prior to it, and won't run until that command
+#   has passed (though not necessarily within the same run. See the
+#   "dependencies" option under the "command:" category, below).
+# - "server" specifies the server on which the commands will be run. Kermit
+#   will try to ssh to this server as each user, using keys uploaded via the
+#   key interface.
+# - "fingerprints" specifies a list (although one is usually sufficient) of
+#   ssh fingerprints for the destination server. This can be set to the special
+#   string 'ignore' if fingerprint checking should be disabled - though this
+#   is not recommended.
+#
+# Within 'workspace' sections, the following items have meaning:
+#
+# - "username" is the name of the user that owns the workspace. This is used
+#   to communicate with the right kermit process. Note that if left blank, the
+#   workspace is assumed to be owned by the user owning the commit.
+# - "setup_cmds" and "abort_cmds" are lists of commands used to prepare a
+#   shared workspace, or tear it down respectively. These commands are only
+#   run when necessary, which is to say when the "pre_cmds" for the workspace
+#   fail. The intention is to run expensive commands here, so that commits
+#   making use of these workspaces can complete faster. Note that if "pre_cmds"
+#   fails, forcing a rerun of the "setup_cmds", then the "pre_cmds" are not
+#   run as well (i.e. "setup_cmds" is expected to perform the actions of
+#   "pre_cmds" if they are necessary).
+# - "pre_cmds" and "post_cmds" are lists of commands to run before/after any
+#   commit commands that have been offloaded to this workspace. "pre_cmds"
+#   should include any items that are needed to prepare the workspace, and
+#   make sure it is still valid (e.g. up to date). If any of them fail, it
+#   will cause the workspaces to be torn down and set up from scratch.
+#   "post_cmds" will be run no matter what happens (completion of all steps,
+#   failure of a step, or abort of the whole commit), and must leave the
+#   workspace ready for the next commit.
+#
+# Within 'command' sections, the following items have meaning:
+#
+# - "command" is the command line to run. This is parsed using shell parsing
+#   rules, so quotes can be used to join space-separated words into single
+#   tokens.
+# - "pwd" is the working directory to run the command in, relative to the
+#   base directory of the target. Note that the working directory may not
+#   be higher than the base directory of the target (with the default being
+#   "./", i.e. the target directory).
+# - "dependencies" is a list of other commands that must always be run in the
+#   workspace prior to this command. These commands will be run, even if they
+#   have previously succeded for the patch in question and normally would
+#   persist their results.
+# - "persist" is a boolean indicating whether the results of this command can
+#   be kept from run to run (assuming the checksum of the patch to commit
+#   does not change between runs). Default is no.
+# - "attempts" is the number of times to attempt the command. This can be
+#   used for commands that are known to be flakey, or fail intermittently.
+#   Default is 1.
+# - "attempt_interval" is the number of seconds to wait between retries of
+#   this command. Default is 10.
+# - "workspace" is the name of the shared workspace to offload this command
+#   to. This will only happen if the target lists this workspace as a valid
+#   one in its "workspaces" configuration option.
+# - "environment" is a list of "key: value" pairs that are set in the
+#   environment of the command when spawned. Default is to inherit the env
+#   of the kermit process. Note that these variables are added (or
+#   overwrite) the starting environment, and do not replace it.
+# - "filevars" is a list of "varname: filename" pairs that describe
+#   interpolation variables (see below) whose values should be the contents
+#   of the given files. See the section on value interpolation below for the
+#   full semantics of this option.
+# - "savelogs" is a list of files that should be saved into the log directory.
+#   Usually these are output files generated in some temporary directory, and
+#   that will be either useful to the user, or required in a later step. The
+#   file name in the logdir is the basename of the input file.
+#
+# Certain options in the "command:" sections can contain interpolation
+# variables (using Python's .format and attr-based lookups; i.e. {varname}).
+# The values available are looked up in order from the following:
+#
+# - Any "filevars" defined in the section.
+# - Any options defined in the "command:" section.
+# - Any options defined in the corresponding "workspace:" section. This only
+#   applies to operations running in a shared workspace.
+# - Any options defined in the corresponding target section.
+# - Various global parameters (see below).
+#
+# Note that options here are not restricted to options understood by kermit.
+# Any options may be specified in the config file, and they will be used in
+# interpolation, allowing re-usable command snippets to be created, and
+# tweaked as needed for each target.
+#
+# The following values are interpolated:
+# - The filename portion of any "filevars". Note that, obviously, the
+#   interpolation here only includes variables already loaded. Variables are
+#   set in the order defined, so earlier file contents can be used in the names
+#   of later filevars, if required.
+# - The values of any environment variables.
+# - The command line (after lexical parsing of the command line into separate
+#   tokens. i.e. any interpolated variable can only be a single token).
+# - The working directory for the command ("pwd").
+# - The paths to any files in the "savelogs" directive.
+#
+# The following global configuration options are always present and usable
+# (though they can be overriden in configuration sections if desired):
+#
+# - "logdir": the directory into which logs should be/are written.
+# - "target": the current target.
+# - "key": the unique identifier for this kermit operation.
+# - "commitdiff": path to the file being committed.
+# - "buglist": the list of bug IDs being committed against.
+# - "commitmsg": the (optional) commit message specified by the user.
+# - "username": the username of the user the commit is for (note: this is not
+#   necessarily the user the command is running as, i.e. in the case of a
+#   shared workspace running as a separate user being used to run part of the
+#   commit).
+# - "stages": list of the stages requested by the user (note: this may not
+#   be exactly the stages being run, since the user may not have correctly
+#   taken dependencies into account).
+#
+# There are also some cosmetic-only configuration options, used to customise
+# the web interface:
+#
+# - "message" is configurable under the 'target' section, is interpolated (with
+#   options from the target and global sections only), and is displayed within
+#   the commit panel on the web interface. An example might be to let the user
+#   know where log files can be accessed.
+# - "description" is configured under the 'command' sections, and replaces the
+#   short command name wherever the command is mentioned in the web interface.
+# - "recovery" is also configured under the 'command' sections, and is shown
+#   in an alert within the command panel whenever the command fails.

=== renamed file 'doc/EnDroid.png' => 'resources/httpinterface/EnDroid.png'
=== modified file 'resources/httpinterface/index.html'
--- resources/httpinterface/index.html	2014-04-05 11:42:51 +0000
+++ resources/httpinterface/index.html	2014-09-01 20:21:07 +0000
@@ -5,7 +5,7 @@
   </head>
   <body bgcolor="white">
     <h1>EnDroid web interface</h1>
-    <img style='float: right' src='_media/EnDroid.png'>
+    <img style='float: right' src='_media/httpinterface/EnDroid.png'>
     {% if plugins %}
     <p>The following plugins have resources registered:</p>
     <ul>

=== added directory 'resources/kermit'
=== added file 'resources/kermit/alert.css'
--- resources/kermit/alert.css	1970-01-01 00:00:00 +0000
+++ resources/kermit/alert.css	2014-09-01 20:21:07 +0000
@@ -0,0 +1,8 @@
+/*
+ * Styles for the global alert service.
+ */
+
+.alerts {
+  position: fixed;
+  z-index: 100;
+}

=== added file 'resources/kermit/alert.js'
--- resources/kermit/alert.js	1970-01-01 00:00:00 +0000
+++ resources/kermit/alert.js	2014-09-01 20:21:07 +0000
@@ -0,0 +1,31 @@
+/*
+ * Implements a global alert service in angular.
+ */
+angular.module("endroid.kermit.alert", []).factory('alertService',
+  ['$rootScope', '$timeout', function($rootScope, $timeout) {
+    $rootScope.alerts = [];
+    var alertService = {
+      add: function(type, msg, timeout) {
+        $rootScope.alerts.push({
+          type: type,
+          msg: msg,
+          close: function() {
+            return alertService.closeAlert(this);
+          }
+        });
+        var idx = $rootScope.alerts.length - 1;
+        if (timeout) {
+          $timeout(function() {
+            alertService.closeAlertIdx(idx);
+          }, timeout);
+        }
+      },
+      closeAlert: function(thisAlert) {
+        return this.closeAlertIdx($rootScope.alerts.indexOf(thisAlert));
+      },
+      closeAlertIdx: function(index) {
+        return $rootScope.alerts.splice(index, 1);
+      }
+    };
+    return alertService;
+  }]);

=== added file 'resources/kermit/kermit-icon-120.png'
Binary files resources/kermit/kermit-icon-120.png	1970-01-01 00:00:00 +0000 and resources/kermit/kermit-icon-120.png	2014-09-01 20:21:07 +0000 differ
=== added file 'resources/kermit/kermit-icon-152.png'
Binary files resources/kermit/kermit-icon-152.png	1970-01-01 00:00:00 +0000 and resources/kermit/kermit-icon-152.png	2014-09-01 20:21:07 +0000 differ
=== added file 'resources/kermit/kermit-icon-60.png'
Binary files resources/kermit/kermit-icon-60.png	1970-01-01 00:00:00 +0000 and resources/kermit/kermit-icon-60.png	2014-09-01 20:21:07 +0000 differ
=== added file 'resources/kermit/kermit-icon-76.png'
Binary files resources/kermit/kermit-icon-76.png	1970-01-01 00:00:00 +0000 and resources/kermit/kermit-icon-76.png	2014-09-01 20:21:07 +0000 differ
=== added file 'resources/kermit/kermit-icon.png'
Binary files resources/kermit/kermit-icon.png	1970-01-01 00:00:00 +0000 and resources/kermit/kermit-icon.png	2014-09-01 20:21:07 +0000 differ
=== added file 'resources/kermit/kermit-logo.jpg'
Binary files resources/kermit/kermit-logo.jpg	1970-01-01 00:00:00 +0000 and resources/kermit/kermit-logo.jpg	2014-09-01 20:21:07 +0000 differ
=== added file 'resources/kermit/kermit.css'
--- resources/kermit/kermit.css	1970-01-01 00:00:00 +0000
+++ resources/kermit/kermit.css	2014-09-01 20:21:07 +0000
@@ -0,0 +1,57 @@
+/*
+ * Common styles for all kermit pages.
+ */
+
+.nav, .pagination, .carousel, .panel-title a { cursor: pointer; }
+
+.navbar-form .input-group-btn,
+.navbar-form .input-group-addon {
+  width: auto;
+}
+
+.kermit-logo {
+  position: absolute;
+  right: 0;
+  z-index: -1;
+}
+
+@media (max-width: 480px) {
+  .navbar { margin-bottom: 5px; }
+  h1 {
+    font-size: 210%;
+    margin-top: 0px;
+    margin-bottom: 0px;
+  }
+  .container > .navbar-header {
+    margin-right: 0;
+  }
+  .container > .navbar-header > button {
+    margin-right: 0;
+  }
+  .container { padding: 0 5px; }
+  .kermit-logo { display: none; }
+  .navbar-form { border: 0; }
+}
+
+/* Animations for alerts */
+.kermit-alert {
+  width: 600px;
+}
+.kermit-alert.ng-enter,
+.kermit-alert.ng-leave {
+  -webkit-transition: opacity 1s;
+  -moz-transition: opacity 1s;
+  -ms-transition: opacity 1s;
+  -o-transition: opacity 1s;
+  transition: opacity 1s;
+}
+.kermit-alert.ng-leave,
+.kermit-alert.ng-enter.ng-enter-active {
+  opacity: 1;
+}
+.kermit-alert.ng-enter,
+.kermit-alert.ng-leave.ng-leave-active {
+  opacity: 0;
+}
+
+

=== added file 'resources/kermit/kermit.png'
Binary files resources/kermit/kermit.png	1970-01-01 00:00:00 +0000 and resources/kermit/kermit.png	2014-09-01 20:21:07 +0000 differ
=== added file 'resources/kermit/kermit_splash.png'
Binary files resources/kermit/kermit_splash.png	1970-01-01 00:00:00 +0000 and resources/kermit/kermit_splash.png	2014-09-01 20:21:07 +0000 differ
=== added file 'resources/kermit/profile.css'
--- resources/kermit/profile.css	1970-01-01 00:00:00 +0000
+++ resources/kermit/profile.css	2014-09-01 20:21:07 +0000
@@ -0,0 +1,64 @@
+/*
+ * Styles for the kermit queue page.
+ */
+
+.profile-toolbar {
+  min-height: 1em;
+}
+
+.profile-toolbar > .btn {
+  margin-top: -2em;
+  margin-bottom: 0.5em;
+  margin-left: 2px;
+}
+
+/* Main queue width control */
+@media (min-width: 768px) {
+  .queue-display {
+    max-width: 750px;
+    padding-right: 250px;
+  }
+}
+@media (min-width: 992px) {
+  .queue-display {
+    max-width: 750px;
+    padding-right: 138px; /* 280 - (992-750)/2 */
+  }
+}
+@media (min-width: 1200px) {
+  .queue-display {
+    max-width: 750px;
+    padding-right: 55px; /* 280 - (1200-750)/2 */
+  }
+}
+
+/* Animations for the list of commits */
+.key-entry.ng-hide-add,
+.key-entry.ng-hide-remove,
+.key-entry.ng-enter,
+.key-entry.ng-leave
+{
+  -webkit-transition: 400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all;
+  -moz-transition: 400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all;
+  -ms-transition: 400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all;
+  -o-transition: 400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all;
+  transition: 400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all;
+
+  position: relative;
+  display: block !important;
+}
+.key-entry.ng-hide-remove.ng-hide-remove-active,
+.key-entry.ng-hide-add,
+.key-entry.ng-enter.ng-enter-active,
+.key-entry.ng-leave {
+  opacity: 1;
+  left: 0;
+}
+.key-entry.ng-hide-add.ng-hide-add-active,
+.key-entry.ng-hide-remove,
+.key-entry.ng-leave.ng-leave-active,
+.key-entry.ng-enter {
+  opacity: 0;
+  left: -800px;
+}
+

=== added file 'resources/kermit/profile.html'
--- resources/kermit/profile.html	1970-01-01 00:00:00 +0000
+++ resources/kermit/profile.html	2014-09-01 20:21:07 +0000
@@ -0,0 +1,147 @@
+<!DOCTYPE html>
+<html ng-app="endroid">
+  <head>
+    <title>User Keys - {{ username | escape }}</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
+    <script src='http://ajax.googleapis.com/ajax/libs/angularjs/1.2.8/angular.min.js'></script>
+    <script src='http://ajax.googleapis.com/ajax/libs/angularjs/1.2.8/angular-animate.min.js'></script>
+    <script src='http://angular-ui.github.io/bootstrap/ui-bootstrap-tpls-0.10.0.js'></script>
+    <script src='/_media/kermit/alert.js'></script>
+    <script src='/_media/kermit/services.js'></script>
+    <script src='/_media/kermit/profile.js'></script>
+
+    <link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css">
+    <link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap-theme.min.css">
+    <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+
+    <link rel="stylesheet" href="/_media/kermit/kermit.css">
+    <link rel="stylesheet" href="/_media/kermit/alert.css">
+    <link rel="stylesheet" href="/_media/kermit/profile.css">
+
+    <link rel="icon" href='/_media/kermit/kermit.png' type='image/png'>
+
+    <link rel="apple-touch-icon" href="/_media/kermit/kermit-icon-60.png">
+    <link rel="apple-touch-icon" sizes="76x76" href="/_media/kermit/kermit-icon-76.png">
+    <link rel="apple-touch-icon" sizes="120x120" href="/_media/kermit/kermit-icon-120.png">
+    <link rel="apple-touch-icon" sizes="152x152" href="/_media/kermit/kermit-icon-152.png">
+    <link rel="apple-touch-startup-image" href="/_media/kermit/kermit_splash.png">
+    <meta name="apple-mobile-web-app-capable" content="yes">
+    <meta name="apple-mobile-web-app-status-bar-style" content="black">
+  </head>
+
+  <body>
+    <script>
+      username = "{{ username | escape }}";
+      initial_keys = {{ keys }};
+    </script>
+
+    <nav class="navbar navbar-static-top navbar-inverse" role="navigation">
+      <div class="container">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle" ng-init='navCollapse = true' ng-click='navCollapse = !navCollapse'>
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" href="/">EnDroid</a>
+        </div>
+        <div class='collapse navbar-collapse' collapse='navCollapse'>
+          <ul class="nav navbar-nav">
+            <li><a href="/kermit/">Kermit Queue</a></li>
+            <!--<li class="dropdown">
+              <a class="dropdown-toggle">Actions <b class="caret"></b></a>
+              <ul class="dropdown-menu">
+                <li><a>The first action</a></li>
+                <li><a>A second action</a></li>
+                <li class="divider"></li>
+                <li><a>A final action</a></li>
+              </ul>
+            </li>-->
+          </ul>
+          <!--<form class="navbar-form navbar-right" role="search">
+            <div class="input-group">
+              <input type="text" class="form-control" placeholder="Search">
+              <span class="input-group-btn">
+                <button class="btn btn-default" type="submit">
+                  <span class="glyphicon glyphicon-search"></span>
+                </button>
+              </span>
+            </div>
+          </form>-->
+          <p class="navbar-text navbar-right">Signed in as <a href="/kermit/profile" class="navbar-link">{{ username }}</a></p>
+        </div>
+      </div>
+    </nav>
+
+    <img class="kermit-logo" src='/_media/kermit/kermit-logo.jpg'>
+
+    <div class="container alerts">
+      <alert ng-repeat="alert in alerts" type="alert.type" class="pull-right kermit-alert" close="alert.close()">{{ '{{' }} alert.msg }}</alert>
+    </div>
+
+    <div class="container">
+      <div ng-controller="ProfileDisplay" class='queue-display' >
+        <script type='text/ng-template' id='uploadModel.html'>
+<div class='modal-header'>
+  <h3>Upload existing Private Key</h3>
+</div>
+<div class='modal-body'>
+  <div>
+    <div class='form-horizontal'>
+      <alert type="'warning'">
+        <i class='fa fa-warning fa-2x pull-left'></i>
+        Note that you must upload your full private key, so please consider the security implications of uploading it over the network. It might be best to let Kermit generate a new key for you, so only the public key is transmitted.</alert>
+      <div class='form-group'>
+        <label class='col-md-3 control-label' for='privatekey'>Private Key</label>
+        <div class='col-md-8'>
+          <textarea class='form-control' ng-model='info.privatekey' id='privatekey' name='privatekey' rows='5'></textarea>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+<div class='modal-footer'>
+  <button class='btn btn-primary' ng-click='ok()'>Upload</button>
+  <button class='btn btn-danger' ng-click='cancel()'>Cancel</button>
+</div>
+        </script>
+        <h1>User Keys</h1>
+
+        <div class='profile-toolbar'>
+          <button class='btn btn-default pull-right' ng-click='generate()' tooltip='Generate a new key'>
+            <i class='fa fa-key' ng-class="{'fa-spin': generating}"></i>
+          </button>
+          <button class='btn btn-default pull-right' ng-click='open()' tooltip='Upload an existing key'>
+            <span class='fa fa-upload'></span>
+          </button>
+        </div>
+
+        <alert ng-show="keys.length == 0" type="'info'">
+          <i class='fa fa-unlock pull-left fa-2x'></i>
+          No keys currently configured.
+        </alert>
+
+        <alert ng-show="keys.length != 0" type="'info'">
+          <i class='fa fa-wrench pull-left fa-2x'></i>
+          Place one of the public keys listed below into the
+          <code>.ssh/authorized_keys</code> file on each server that you may
+          use for commits.
+        </alert>
+
+        {% raw %}
+        <div class='panel panel-default key-entry' ng-repeat='key in keys'>
+          <div class='panel-heading'>
+            {{ key.type }} Key
+            <button type="button" class="pull-right close" aria-hidden="true" ng-click='remove($index)'>&times;</button>
+          </div>
+          <div class='panel-body'>
+            <pre style='white-space: pre-wrap'>{{ key.keystr }}</pre>
+          </div>
+        </div>
+        {% endraw %}
+
+      </div>
+    </div>
+  </body>
+</html>

=== added file 'resources/kermit/profile.js'
--- resources/kermit/profile.js	1970-01-01 00:00:00 +0000
+++ resources/kermit/profile.js	2014-09-01 20:21:07 +0000
@@ -0,0 +1,77 @@
+/*
+ * Main controllers, etc. for the kermit queue page.
+ */
+var module = angular.module("endroid", ["ui.bootstrap", "ngAnimate",
+                                        "endroid.kermit.alert", "asyncData"]);
+
+function ProfileDisplay($scope, $modal, alertService, asyncData, $http) {
+  $scope.keys = initial_keys;
+  $scope.generating = false;
+
+  $scope.generate = function() {
+    $scope.generating = true;
+    asyncData.getData("generatekey").then(function(data) {
+      $scope.keys.push(data.success);
+      $scope.generating = false;
+    }, function(data) {
+      $scope.generating = false;
+      alertService.add('danger', "Failed to generate a new key: " + data.fail,
+                       5000);
+    });
+  };
+
+  $scope.open = function() {
+    var modalInstance = $modal.open({
+      templateUrl: 'uploadModel.html',
+      controller: ModalInstanceCtrl,
+    });
+
+    modalInstance.result.then(function(result) {
+      alertService.add('success', "Key uploaded successfully", 5000);
+      $scope.keys.push(result);
+    }, function(failure) {
+      alertService.add('danger', "Failed to upload key: " + failure, 5000);
+    });
+  };
+
+  $scope.remove = function(idx) {
+    $http({
+      method: "GET",
+      url: "removekey?idx=" + $scope.keys[idx].idx,
+    }).then(function(data) {
+      $scope.keys.splice(idx, 1);
+    }, function(data) {
+      alertService.add('danger', "Failed to delete key: " + data.fail, 5000);
+    });
+  };
+};
+
+var ModalInstanceCtrl = function($scope, $modalInstance, $http) {
+  $scope.info = {
+    privatekey: '',
+  };
+
+  $scope.cancel = function() {
+    $modalInstance.dismiss("cancelled by you");
+  };
+
+  $scope.ok = function() {
+    $http({
+      method: "POST",
+      url: "uploadkey",
+      transformRequest: function(obj) {
+        var str = [];
+        angular.forEach(obj, function(v, k) {
+          str.push(encodeURIComponent(k) + "=" + encodeURIComponent(v));
+        });
+        return str.join("&");
+      },
+      data: $scope.info,
+      headers: {"Content-Type": 'application/x-www-form-urlencoded'}
+    }).then(function(data) {
+      $modalInstance.close(data.data.success);
+    }, function(data) {
+      $modalInstance.dismiss(data.data.fail);
+    });
+  };
+};

=== added file 'resources/kermit/queue.css'
--- resources/kermit/queue.css	1970-01-01 00:00:00 +0000
+++ resources/kermit/queue.css	2014-09-01 20:21:07 +0000
@@ -0,0 +1,169 @@
+/*
+ * Styles for the kermit queue page.
+ */
+
+.queue-toolbar {
+  min-height: 10px;
+}
+
+.queue-toolbar > .btn {
+  margin-top: -30px;
+  margin-bottom: 5px;
+  margin-left: 2px;
+}
+
+.filter-bar .navbar-form {
+  padding: 0;
+  border: none;
+}
+
+.filter-bar .watermark {
+  opacity: 0.1;
+  position: absolute;
+  right: 0;
+  margin: 0.1em;
+}
+
+.active-filter {
+  position: relative;
+  display: inline-block;
+  margin-right: 2px;
+  margin-top: 2px;
+  border-radius: 0.25em;
+  color: #ffffff;
+  padding: 0.2em 0.2em 0.2em 0.6em;
+  font-size: 90%;
+  float: left;
+}
+
+.active-filter button.close {
+  font-size: 140%; 
+  padding-left: 2px;
+}
+
+.active-filter.ng-enter,
+.active-filter.ng-leave
+{
+  -webkit-transition: 500ms all;
+  -moz-transition: 500ms all;
+  -ms-transition: 500ms all;
+  -o-transition: 500ms all;
+  transition: 500ms all;
+
+  position: relative;
+  display: inline-block !important;
+}
+.active-filter.ng-enter.ng-enter-active,
+.active-filter.ng-leave {
+  left: 0;
+  opacity: 1;
+}
+.active-filter.ng-leave.ng-leave-active,
+.active-filter.ng-enter {
+  left: -300px;
+  opacity: 0;
+}
+
+@media (max-width: 480px) {
+  div.commit-entry.panel {
+    margin: 0 -5px;
+    padding: 0;
+    border-left: 0;
+    border-right: 0;
+    border-radius: 0;
+    -webkit-border-radius: 0;
+  }
+  .commit-entry .list-group { margin: 0 -15px; }
+  .commit-entry .panel-heading {
+    border-radius: 0;
+    -webkit-border-radius: 0;
+    padding: 10px 5px;
+  }
+  .commit-entry .panel-body {
+    padding: 15px 5px;
+  }
+}
+
+/* Main queue width control */
+@media (min-width: 768px) {
+  .queue-display {
+    max-width: 750px;
+    padding-right: 250px;
+  }
+}
+@media (min-width: 992px) {
+  .queue-display {
+    max-width: 750px;
+    padding-right: 138px; /* 280 - (992-750)/2 */
+  }
+}
+@media (min-width: 1200px) {
+  .queue-display {
+    max-width: 750px;
+    padding-right: 55px; /* 280 - (1200-750)/2 */
+  }
+}
+
+/* Animations for the filter bar */
+.filter-bar.ng-hide-remove,
+.filter-bar.ng-hide-add {
+  -webkit-transition: all 0.5s;
+  -moz-transition: all 0.5s;
+  -ms-transition: all 0.5s;
+  -o-transition: all 0.5s;
+  transition: all 0.5s;
+
+  position: relative;
+  display: block !important;
+}
+.filter-bar.ng-hide-remove,
+.filter-bar.ng-hide-add.ng-hide-add-active {
+  opacity: 0;
+  height: 0px;
+  min-height: 0px;
+  margin-bottom: -2px;
+}
+.filter-bar.ng-hide-remove.ng-hide-remove-active,
+.filter-bar.ng-hide-add {
+  opacity: 1;
+  height: 87px;
+  min-height: 87px;
+  margin-bottom: 20px;
+}
+.filter-bar.ng-hide-add > div,
+.filter-bar.ng-hide-remove > div {
+  display: none !important;
+}
+
+/* Animations for the list of commits */
+.commit-entry.ng-hide-add,
+.commit-entry.ng-hide-remove,
+.commit-entry.ng-enter,
+.commit-entry.ng-leave
+{
+  -webkit-transition: 400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all;
+  -moz-transition: 400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all;
+  -ms-transition: 400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all;
+  -o-transition: 400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all;
+  transition: 400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all;
+
+  position: relative;
+  display: block !important;
+}
+.commit-entry.ng-hide-remove.ng-hide-remove-active,
+.commit-entry.ng-hide-add,
+.commit-entry.ng-enter.ng-enter-active,
+.commit-entry.ng-leave {
+  opacity: 1;
+  left: 0;
+  height: 38px;
+}
+.commit-entry.ng-hide-add.ng-hide-add-active,
+.commit-entry.ng-hide-remove,
+.commit-entry.ng-leave.ng-leave-active,
+.commit-entry.ng-enter {
+  opacity: 0;
+  left: -800px;
+  height: 0px;
+}
+

=== added file 'resources/kermit/queue.html'
--- resources/kermit/queue.html	1970-01-01 00:00:00 +0000
+++ resources/kermit/queue.html	2014-09-01 20:21:07 +0000
@@ -0,0 +1,305 @@
+<!DOCTYPE html>
+<html ng-app="endroid">
+  <head>
+    <title>Kermit Queue</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
+    <script src='http://ajax.googleapis.com/ajax/libs/angularjs/1.2.8/angular.js'></script>
+    <script src='http://ajax.googleapis.com/ajax/libs/angularjs/1.2.8/angular-animate.min.js'></script>
+    <script src='http://angular-ui.github.io/bootstrap/ui-bootstrap-tpls-0.10.0.js'></script>
+    <script src='/_media/kermit/alert.js'></script>
+    <script src='/_media/kermit/services.js'></script>
+    <script src='/_media/kermit/queue.js'></script>
+
+    <link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css">
+    <link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap-theme.min.css">
+    <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+
+    <link rel="stylesheet" href="/_media/kermit/kermit.css">
+    <link rel="stylesheet" href="/_media/kermit/alert.css">
+    <link rel="stylesheet" href="/_media/kermit/queue.css">
+
+    <link rel="shortcut icon" href='/_media/kermit/kermit.png' type='image/png'>
+
+    <link rel="apple-touch-icon" href="/_media/kermit/kermit-icon-60.png">
+    <link rel="apple-touch-icon" sizes="76x76" href="/_media/kermit/kermit-icon-76.png">
+    <link rel="apple-touch-icon" sizes="120x120" href="/_media/kermit/kermit-icon-120.png">
+    <link rel="apple-touch-icon" sizes="152x152" href="/_media/kermit/kermit-icon-152.png">
+    <link rel="apple-touch-startup-image" href="/_media/kermit/kermit_splash.png">
+    <meta name="apple-mobile-web-app-capable" content="yes">
+    <meta name="apple-mobile-web-app-status-bar-style" content="black">
+  </head>
+
+  <body>
+    <script>
+      username = "{{ username | escape }}";
+    </script>
+
+    <nav class="navbar navbar-static-top navbar-inverse" role="navigation">
+      <div class="container">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle" ng-init='navCollapse = true' ng-click='navCollapse = !navCollapse'>
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" target="_self" href="/">EnDroid</a>
+        </div>
+        <div class="collapse navbar-collapse" collapse='navCollapse'>
+          <ul class="nav navbar-nav">
+            <li class="active"><a target="_self" href="/kermit/">Kermit Queue</a></li>
+            <!--<li class="dropdown">
+              <a class="dropdown-toggle">Actions <b class="caret"></b></a>
+              <ul class="dropdown-menu">
+                <li><a>The first action</a></li>
+                <li><a>A second action</a></li>
+                <li class="divider"></li>
+                <li><a>A final action</a></li>
+              </ul>
+            </li>-->
+          </ul>
+          <!--<form class="navbar-form navbar-right" role="search">
+            <div class="input-group">
+              <input type="text" class="form-control" placeholder="Search">
+              <span class="input-group-btn">
+                <button class="btn btn-default" type="submit">
+                  <span class="glyphicon glyphicon-search"></span>
+                </button>
+              </span>
+            </div>
+          </form>-->
+          <p class="navbar-text navbar-right">Signed in as <a target="_self" href="profile" class="navbar-link">{{ username }}</a></p>
+        </div>
+      </div>
+    </nav>
+
+    <img class="kermit-logo" src='/_media/kermit/kermit-logo.jpg'>
+
+    <div class="container alerts">
+      <alert ng-repeat="alert in alerts" type="alert.type" class="pull-right kermit-alert" close="alert.close()">{{ '{{' }} alert.msg }}</alert>
+    </div>
+
+    <div class="container">
+    <div ng-controller="QueueDisplay" class='queue-display' >
+      <script type='text/ng-template' id='submitModal.html'>
+<div class='modal-header'>
+  <h3>Submit new Commit to the Queue</h3>
+</div>
+<div class='modal-body'>
+  <div>
+    <tabset>
+      <tab heading="Basic Info">
+        <p></p>
+        <div class="form-horizontal">
+          <input type='hidden' id='username' name='username' ng-model='info.username' value='{{username}}' ng-init="info.username = '{{username|escape}}'">
+          <div class="form-group">
+            <label class="col-md-3 control-label" for="commitdiff">Patch File</label>
+            <div class="col-md-7">
+              <input id="commitdiff" name="commitdiff" placeholder="path to patch file" class="form-control input-md" type="text" ng-model="info.commitdiff">
+            </div>
+          </div>
+          <div class="form-group">
+            <label class="col-md-3 control-label" for="target">Target</label>
+            <div class="col-md-7">
+              <input id="target" name="target" typeahead="target for target in targets | filter:$viewValue | limitTo:8" placeholder="name of target" class="form-control input-md" type="text" ng-model="info.target" autocomplete="off">
+            </div>
+          </div>
+          <div class="form-group">
+            <label class="col-md-3 control-label" for="buglist">Bug IDs</label>
+            <div class="col-md-7">
+              <input id="buglist" name="buglist" placeholder="list of bugs to commit against" class="form-control input-md" type="text" ng-model="info.buglist">
+            </div>
+          </div>
+        </div>
+      </tab>
+      <tab heading="Commit Info">
+        <p></p>
+        <div class="form-horizontal">
+          <div class="form-group">
+            <label class="col-md-3 control-label" for="commitmsg">Commit Message</label>
+            <div class="col-md-7">
+              <textarea class="form-control" ng-model="info.commitmsg" id="commitmsg" name="commitmsg" rows='5'></textarea>
+            </div>
+          </div>
+        </div>
+      </tab>
+      {% raw %}
+      <tab heading="Advanced" disabled="stages.length == 0">
+        <div class='container-fluid'>
+          <div class='col-md-6'>
+            <h4>
+              Control Stages:
+              <i style='margin-left: 1em; cursor: help' class='pull-right glyphicon glyphicon-info-sign' tooltip='Note that dependencies between stages are obeyed, so multiple items may enable/disable at once' tooltip-popup-delay='500' tooltip-placement='left'></i>
+            </h4>
+            <div class="btn-group-vertical" style='width: 100%'>
+              <button ng-repeat="stage in stages"
+                      class="btn btn-default col-md-5"
+                      type="button"
+                      kermit-name='{{ stage.name }}'
+                      ng-class="{'btn-danger': info.stages.indexOf('{{ stage.name }}') == -1}"
+                      ng-click="updateStages($event.target)">
+                {{ stage.name }}
+              </button>
+            </div>
+          </div>
+          <div class='col-md-6'>
+            <h4>
+              Control Workspaces:
+              <i style='margin-left: 1em; cursor: help' class='pull-right glyphicon glyphicon-info-sign' tooltip='Control whether shared workspaces are used, and whether to reuse them in their current state or set them up from scratch' tooltip-popup-delay='500' tooltip-placement='left'></i>
+            </h4>
+            <div class="btn-group-vertical" style='width: 100%'>
+              <button ng-repeat="workspace in workspaces"
+                      class="btn btn-default col-md-5 btn-success"
+                      type="button"
+                      kermit-name='{{ workspace }}'
+                      tooltip="{{ workspaceTooltip(workspace) }}"
+                      ng-class="{'btn-danger': info.ws_disable.indexOf('{{ workspace }}') != -1, 'btn-warning': info.ws_reset.indexOf('{{ workspace }}') != -1}"
+                      ng-click="updateWorkspaces($event.target)">
+                {{ workspace }}
+              </button>
+            </div>
+        </div>
+      </tab>
+      {% endraw %}
+    </tabset>
+  </div>
+</div>
+<div class='modal-footer'>
+  <button class='btn btn-primary' ng-click='ok()'>Submit</button>
+  <button class='btn btn-danger' ng-click='cancel()'>Cancel</button>
+</div>
+      </script>
+
+      <h1>Kermit Queue</h1>
+
+      <div class='queue-toolbar'>
+        <button class='btn btn-default pull-right' ng-click='open()'>
+          <span class='glyphicon glyphicon-plus'></span>
+        </button>
+        <button class='btn btn-default pull-right' ng-class='{"btn-info": active_filters.length > 2 || active_filters[0].type != "username" || active_filters[0].value != username, "active": filter_shown}' ng-click='filter_shown = !filter_shown; filter_class = filter_shown ? "active" : ""'> 
+          <span class='glyphicon glyphicon-filter'></span>
+        </button>
+        <div class='btn pull-right' ng-show='update_pending'>
+          <span class='fa fa-spinner fa-spin'></span>
+        </div>
+      </div>
+
+      {% raw %}
+      <script type="text/ng-template" id="filter_template.html">
+        <a tooltip='{{ match.model.type | capitalise }}' tooltip-placement='left'>
+          <i class='fa fa-fw' ng-class='{"fa-user": match.model.type == "username", "fa-tasks": match.model.type == "state", "fa-dot-circle-o": match.model.type == "target", "fa-bug": match.model.type == "buglist", "fa-thumb-tack": match.model.type == "key"}'></i>
+          {{ match.label }}
+        </a>
+      </script>
+
+      <div class='navbar navbar-default filter-bar' ng-show='filter_shown'>
+        <div class='container-fluid'>
+          <i class='glyphicon glyphicon-filter fa-5x fa-fw watermark'></i>
+          <form class='navbar-form'>
+            <p>
+              <span ng-repeat='filter in active_filters | filter:{type: "!lastmod"}' class='active-filter label-default' ng-class='"label-" + filter_type_colours[filter.type]'>
+                <button type="button" class="close" ng-show="filter.type != 'lastmod'" aria-hidden="true" ng-click='remove_filter($index)'>&times;</button>
+                <i class='fa' ng-class='filter_type_classes[filter.type]'></i>
+                <span ng-show='filter.type != "lastmod"'>{{ filter.value }}</span>
+              </span><span class='active-filter label-default' ng-init='datefilter = active_filters[active_filters.length - 1]'>
+                <i class='fa fa-clock-o'></i>
+                <span datepicker-popup='mediumDate' close-text='Close' show-weeks='false' show-button-bar='false' starting-day='1' is-open='lastmod_open' ng-click='lastmod_open = !lastmod_open' datepicker-append-to-body='true' ng-model='datefilter.value' tooltip-append-to-body='true' tooltip='Show only jobs updated since...' style='padding-right: 0.3em; cursor: pointer'>{{ datefilter.value | date : "mediumDate" }}</span>
+              </span>
+            </p>
+            <p style='clear: both'></p>
+            <!-- Add new filter bar -->
+            <div class='form-group'>
+              <div class='input-group'>
+                <div class='input-group-btn'>
+                  <button type='button' class='btn btn-default dropdown-toggle'>
+                    <i class='fa fa-fw' ng-class='filter_type_classes[filtertype] + " " + "text-" + filter_type_colours[filtertype]'></i>
+                  </button>
+                  <ul class='dropdown-menu'>
+                    <li ng-repeat='type in filter_types'>
+                      <a ng-click='set_filter_type(type)' style='padding-left: 10px; cursor: pointer'><i class='fa fa-fw' ng-class='filter_type_classes[type] + " " + "text-" + filter_type_colours[type]'></i> {{ type | capitalise }}</a>
+                    </li>
+                  </ul>
+                </div>
+                <input type='text' ng-model='filter_value' placeholder='Add another filter' typeahead="opt as opt.value for opt in get_filter_options($viewValue) | limitTo:8" class='form-control' typeahead-template-url="filter_template.html" typeahead-wait-ms='200' typeahead-on-select="add_filter($item)" typeahead-editable="false">
+              </div>
+            </div>
+          </form>
+        </div>
+      </div>
+      {% endraw %}
+
+      <!-- Job entries -->
+
+      {% raw %}
+      <alert ng-show="jobs_shown == 0" type="'info'">
+        <i class='fa fa-bullhorn pull-left fa-2x'></i>
+        No jobs match the current filters.
+      </alert>
+      <accordion close-others="false">
+        <accordion-group ng-repeat="job in jobs track by job.key" class="commit-entry" ng-show="matches_filter(job)" is-open='isopen'>
+          <accordion-heading>
+            <div>
+              <i class="pull-right glyphicon" ng-class="{'glyphicon-chevron-down': isopen, 'glyphicon-chevron-right': !isopen}"></i>
+              <span class='pull-right' style='font-variant: small-caps; margin-right: 1em'>{{ job.status_string }}</span>
+              <span>Commit {{ job.status.inputargs.buglist }} to {{ job.status.inputargs.target }} as {{ job.status.inputargs.username }} ({{ job.key }})</span>
+            </div>
+          </accordion-heading>
+          <p ng-click='debug = !debug' style='margin-left: -16px; position: absolute; width: 16px; height: 16px'></p>
+            <button ng-show='job.status_string == "Running" || job.status_string == "Queued"' class='btn btn-danger pull-right' tooltip='Abort Operation' ng-click='abort($event, job.key)'>
+              <i class='glyphicon glyphicon-trash'></i>
+            </button>
+            <button ng-show='job.status_string == "Failed" || job.status_string == "Aborted"' class='btn btn-danger pull-right' tooltip='Resubmit Request' ng-click='resubmit($event, job.key)'>
+              <i class='fa fa-rotate-right' ng-class="{'fa-spin': resubmitting == job.key}"></i>
+            </button>
+          <p>Committing diff {{ job.status.inputargs.commitdiff }}
+            <span ng-show='job.status.currstatus.begin_at || job.status.currstatus.started_at'>(started at {{ (job.status.currstatus.begin_at || job.status.currstatus.started_at) * 1000 | date : 'HH:mm, MMM d' }})</span>
+            <span ng-show='job.status.currstatus.failed_at.keys().length > 0 && !job.status.currstatus.started_at'>(failed preliminary checks at {{ job.status.currstatus.failed_at * 1000 | date : 'HH:mm, MMM d' }})</span>
+          </p>
+          <p ng-show='job.status.currstatus.message'>
+            {{ job.status.currstatus.message }}
+          </p>
+          <alert type="'warning'" collapse="!debug">
+            <pre style='white-space: pre-wrap'>{{ job.status }}</pre>
+          </alert>
+          <alert type="'danger'" ng-show="job.status.currstatus.check_failed">
+            <i class='fa fa-warning pull-left fa-2x'></i>
+            {{ job.status.currstatus.check_failed }}
+          </alert>
+          <alert type="'danger'" ng-repeat="fail in job.status.currstatus.failed">
+            <i class='fa fa-warning pull-left fa-2x'></i>
+            <a target="_tab" href='logfile?key={{job.key}}&amp;stage={{fail}}' class='fa fa-file-text-o fa-2x pull-right' tooltip='View Log file (new window/tab)'></a>
+            {{ job.config.commands[fail].description || fail }} failed. {{ job.config.commands[fail].recovery }}
+          </alert>
+          <h4 ng-show='job.status.currstatus.stages'>Stages:</h4>
+          <ul class="list-group" ng-show='job.status.currstatus.stages'>
+            <li class='list-group-item' ng-class="{'list-group-item-danger': !job.status.currstatus.started_at && job.status.currstatus.failed.length > 0, 'list-group-item-success': job.status.currstatus.started_at, 'list-group-item-info': job.status.currstatus.running.length > 0 && !job.status.currstatus.started_at}">
+              <span>Running preliminary checks</span>
+              <div ng-show="job.status.currstatus.running.length > 0 && !job.status.currstatus.started_at && job.status.currstatus.failed.length == 0" class="progress progress-striped active pull-right" style="width: 30%">
+                <div class="progress-bar progress-bar-info" role="progressbar" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width: 100%" tooltip="{{ job.config.commands[job.status.currstatus.running[0]].description || job.status.currstatus.running[0] }}">
+                </div>
+              </div>
+              <span ng-show="false" class="badge"></span>
+              <span ng-show="job.status.currstatus.started_at" class="badge"><span style="cursor: pointer" class="glyphicon glyphicon-ok" tooltip="{{ job.status.currstatus.started_at * 1000 | date : 'HH:mm, MMM d' }}" tooltip-append-to-body="true"></span></span>
+              <span ng-show="!job.status.currstatus.started_at && job.status.currstatus.failed.length > 0" class="badge"><span style="cursor: pointer" class="glyphicon glyphicon-fire" tooltip="{{ job.status.currstatus.failed_at[job.status.currstatus.failed[0]] * 1000 | date : 'HH:mm, MMM d' }}" tooltip-append-to-body="true"></span></span>
+            </li>
+            <li ng-repeat='stage in job.status.currstatus.stages' class="list-group-item" ng-class="{'list-group-item-danger': job.status.currstatus.failed.indexOf(stage) != -1, 'list-group-item-success': job.status.currstatus.done.indexOf(stage) != -1, 'list-group-item-warning': job.status.currstatus.skipping.indexOf(stage) != -1, 'list-group-item-info': job.status.currstatus.running.indexOf(stage) != -1}">
+              <span tooltip-placement="right" tooltip-popup-delay="200" tooltip="{{ job.config.commands[stage].command }}" style="cursor: help">{{ job.config.commands[stage].description || stage }}</span>
+              <div ng-show="job.status.currstatus.running.indexOf(stage) != -1" class="progress progress-striped active pull-right" style="width: 30%">
+                <div class="progress-bar progress-bar-info" role="progressbar" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width: 100%">
+                </div>
+              </div>
+              <span ng-show="false" class="badge"></span>
+              <span ng-show="job.status.currstatus.done.indexOf(stage) != -1" class="badge"><span style="cursor: pointer" class="glyphicon glyphicon-ok" tooltip="{{ job.status.currstatus.done_at[stage] * 1000 | date : 'HH:mm, MMM d' }}" tooltip-append-to-body="true"></span></span>
+              <span ng-show="job.status.currstatus.failed.indexOf(stage) != -1" class="badge"><span style="cursor: pointer" class="glyphicon glyphicon-fire" tooltip="{{ job.status.currstatus.failed_at[stage] * 1000 | date : 'HH:mm, MMM d' }}" tooltip-append-to-body="true"></span></span>
+              <span ng-show="job.status.currstatus.skipping.indexOf(stage) != -1" class="badge"><span style="cursor: pointer" class="glyphicon glyphicon-flash" tooltip="Skipped as requested by submitter" tooltip-append-to-body="true"></span></span>
+              <span ng-show="job.config.commands[stage].workspace && (!job.status.inputargs.ws_disable || job.status.inputargs.ws_disable.indexOf(job.config.commands[stage].workspace) == -1)" ng-class="{'label-warning': job.status.inputargs.ws_reset && job.status.inputargs.ws_reset.indexOf(job.config.commands[stage].workspace) != -1}" class='label label-primary pull-right' style='margin-right: 5px'><span tooltip="{{ ((job.status.inputargs.ws_reset && job.status.inputargs.ws_reset.indexOf(job.config.commands[stage].workspace) != -1) ? 'Command run in a clean workspace as ' : 'Command run in a separate workspace as ') + job.config.workspaces[job.config.commands[stage].workspace].username }}" tooltip-append-to-body="true" style="cursor: pointer">{{ job.config.commands[stage].workspace }}</span></span>
+            </li>
+          </ul>
+        </accordion-group>
+      </accordion>
+
+      {% endraw %}
+    </div>
+    </div>
+  </body>
+</html>

=== added file 'resources/kermit/queue.js'
--- resources/kermit/queue.js	1970-01-01 00:00:00 +0000
+++ resources/kermit/queue.js	2014-09-01 20:21:07 +0000
@@ -0,0 +1,497 @@
+/*
+ * Main controllers, etc. for the kermit queue page.
+ */
+var module = angular.module("endroid", ["ui.bootstrap", "ngAnimate",
+                                        "endroid.kermit.alert", "asyncData"]);
+
+module.config(['$locationProvider', function($locationProvider) {
+  $locationProvider.html5Mode(true).hashPrefix('!');
+}]);
+
+module.filter('capitalise', function() {
+ return function(input, scope) {
+   if (input != null)
+     input = input.toLowerCase();
+     return input.substring(0, 1).toUpperCase() + input.substring(1);
+   }
+ });
+
+function QueueDisplay($scope, $modal, $timeout, alertService, $http, asyncData,
+                      $filter, $q, $location) {
+  $scope.debug = false;
+  $scope.names = [];
+  $scope.buglists = [];
+  $scope.targets = [];
+  $scope.states = ["Queued", "Running", "Aborted", "Failed", "Completed"];
+  $scope.filter_shown = false;
+  $scope.username = username;
+
+  $scope.known_jobs = {};
+  $scope.jobs = [];
+  $scope.jobs_shown = 0;
+  $scope.last_update = 0;
+  $scope.update_pending = false;
+
+  $scope.load_filter_lists = function() {
+    asyncData.getData("list?arg=user", function(data) {
+      return data.user;
+    }).then(function(names) {
+      $scope.names = names;
+    }, function() {
+      alertService.add('danger', "Failed to fetch known names.",
+                       5000);
+    });
+    asyncData.getData("list?arg=target").then(function(data) {
+      $scope.targets = data.target;
+    }, function() {
+      alertService.add('danger', "Failed to fetch known targets.", 5000);
+    });
+    asyncData.getData("list?arg=buglist").then(function(data) {
+      $scope.buglists = data.buglist;
+    }, function() {
+      alertService.add('danger', "Failed to fetch known bug lists.", 5000);
+    });
+  };
+
+  var filter_query = function(obj) {
+    var str = [];
+    angular.forEach(obj, function(v, k) {
+      str.push(encodeURIComponent(v.type) + "=" + encodeURIComponent(v.value));
+    });
+    console.log(str.join('&'));
+    return str.join('&');
+  };
+
+  var poll = function() {
+      if ($scope.poll_canceler) {
+        $scope.poll_canceler.resolve();
+      }
+      $scope.poll_canceler = $q.defer();
+      asyncData({
+        method: 'POST',
+        url: 'query',
+        transformRequest: function(obj) {
+          var str = filter_query(obj);
+          return 'wait=' + $scope.last_update + '&' + str;
+        },
+        data: $scope.active_filters,
+        timeout: $scope.poll_canceler.promise,
+        headers: {'Content-Type': 'application/x-www-form-urlencoded'}
+      }).then(function(data) {
+        $scope.jobs = data.jobs;
+        $scope.jobs.sort(function(a, b) {
+          if (a.key < b.key) return 1;
+          if (a.key > b.key) return -1;
+          return 0;
+        });
+        $scope.jobs_shown = 0;
+        $scope.last_update = data.last_update;
+        $scope.update_pending = false;
+        poll();
+      }, function(data) {
+        // Wait a bit if the connection failed. Should prob back off.
+        if (data.status != 0) {
+          $timeout(poll, 5000);
+        }
+      });
+  };
+
+  $scope.load_filter_lists();
+
+  var args = $location.search();
+
+  $scope.active_filters = [];
+
+  angular.forEach(args, function(v, k) {
+    if (k == "username" || k == "lastmod" || k == "target" || k == "buglist" ||
+        k == "state") {
+      if (v instanceof Array) {
+        angular.forEach(v, function(e) {
+          if (k == "lastmod") e = new Date(e);
+          $scope.active_filters.push({type: k, value: e});
+        });
+      } else {
+        if (k == "lastmod") v = new Date(v);
+        $scope.active_filters.push({type: k, value: v});
+      }
+    }
+  });
+
+  var sort_filters = function(a, b) {
+    if (a.type != b.type) {
+      if (a.type == "key") return -1;
+      if (b.type == "key") return 1;
+      if (a.type == "username") return -1;
+      if (b.type == "username") return 1;
+      if (a.type == "target") return -1;
+      if (b.type == "target") return 1;
+      if (a.type == "buglist") return -1;
+      if (b.type == "buglist") return 1;
+      if (a.type == "state") return -1;
+      if (b.type == "state") return 1;
+      throw "unexpected objects to compare " + a + ", " + b;
+    } else {
+      if (a.value < b.value) return -1;
+      if (a.value > b.value) return 1;
+      return 0;
+    }
+  };
+
+  $scope.active_filters.sort(sort_filters);
+
+  if ($scope.active_filters.length == 0) {
+    $scope.active_filters.push({type: "username", value: username});
+  }
+  if ($scope.active_filters[$scope.active_filters.length - 1].type != "lastmod") {
+    $scope.active_filters.push({type: "lastmod", value: new Date(new Date().getTime() - 1000 * 60 * 60 * 24 * 14)});
+  }
+
+  $scope.filtertype = "any";
+  $scope.filter_types = [
+    "any", "key", "username", "target", "buglist", "state"
+  ];
+  $scope.filter_type_classes = {
+    any: "fa-asterisk",
+    key: "fa-thumb-tack",
+    username: "fa-user",
+    target: "fa-dot-circle-o",
+    buglist: "fa-bug",
+    state: "fa-tasks",
+    lastmod: "fa-clock-o"
+  };
+  $scope.filter_type_colours = {
+    key: "success",
+    username: "primary",
+    target: "danger",
+    buglist: "warning",
+    state: "info"
+  };
+
+  $scope.set_filter_type = function(type) {
+    console.log("Setting filter to " + type);
+    $scope.filtertype = type;
+  };
+
+  $scope.add_filter = function($item) {
+    $scope.active_filters.push($item);
+    $scope.active_filters.sort(sort_filters);
+    $scope.filter_value = null;
+    $scope.last_update = 0;
+    $scope.update_pending = true;
+    $location.replace();
+    $location.url($location.path() + '?' + filter_query($scope.active_filters));
+    poll();
+  };
+
+  $scope.remove_filter = function(idx) {
+    // Can't remove the date filter
+    if ($scope.active_filters[idx].type == "lastmod") return;
+    $scope.active_filters.splice(idx, 1);
+    $scope.last_update = 0;
+    $scope.update_pending = true;
+    $location.replace();
+    $location.url($location.path() + '?' + filter_query($scope.active_filters));
+    poll();
+  };
+
+  var typematch = function(type) {
+    return type == $scope.filtertype || $scope.filtertype == "any";
+  };
+
+  $scope.get_filter_options = function(val) {
+    var res = [];
+    var sofar = 0;
+    if (!isNaN(val) && typematch("key")) {
+      res.push({type: "key", value: "kermit-" + val})
+    }
+    names = $filter('filter')($scope.names, val);
+    for (var i = 0; i < names.length && typematch("username"); i++) {
+      res.push({type: "username", value: names[i]})
+    } 
+    targets = $filter('filter')($scope.targets, val);
+    for (var i = 0; i < targets.length && typematch("target"); i++) {
+      res.push({type: "target", value: targets[i]})
+    } 
+    states = $filter('filter')($scope.states, val);
+    for (var i = 0; i < states.length && typematch("state"); i++) {
+      res.push({type: "state", value: states[i]})
+    } 
+    bugs = $filter('filter')($scope.buglists, val);
+    for (var i = 0; i < bugs.length && typematch("buglist"); i++) {
+      res.push({type: "buglist", value: bugs[i]})
+    }
+    if (typematch("username")) res.push({type: "username", value: val});
+    if (typematch("target")) res.push({type: "target", value: val});
+    if (typematch("buglist")) res.push({type: "buglist", value: val});
+    return res;
+  };
+
+  var check_filter = function(job) {
+    var username = job.status.inputargs.username;
+    var tgt = job.status.inputargs.target;
+    var buglist = job.status.inputargs.buglist;
+    var state = job.status_string;
+    var lastmod = job.lastmod;
+
+    var matches = true;
+
+    var keyfilters = $filter('filter')($scope.active_filters,
+                                       {type: "key"});
+    if (keyfilters.length > 0) {
+      matches = false;
+      angular.forEach(keyfilters, function(v) {
+        matches = matches || job.key == v.value;
+      });
+    }
+    if (!matches) return matches;
+
+    var userfilters = $filter('filter')($scope.active_filters,
+                                        {type: "username"});
+    if (userfilters.length > 0) {
+      matches = false;
+      angular.forEach(userfilters, function(v) {
+        matches = matches || username.toLowerCase().indexOf(v.value.toLowerCase()) != -1;
+      });
+    }
+    if (!matches) return matches;
+
+    var tgtfilters = $filter('filter')($scope.active_filters,
+                                       {type: "target"});
+    if (tgtfilters.length > 0) {
+      matches = false;
+      angular.forEach(tgtfilters, function(v) {
+        matches = matches || tgt.toLowerCase().indexOf(v.value.toLowerCase()) != -1;
+      });
+    }
+    if (!matches) return matches;
+
+    var bugfilters = $filter('filter')($scope.active_filters, 
+                                       {type: "buglist"});
+    if (bugfilters.length > 0) {
+      matches = false;
+      angular.forEach(bugfilters, function(v) {
+        matches = matches || buglist.toLowerCase().indexOf(v.value.toLowerCase()) != -1;
+      });
+    }
+    if (!matches) return matches;
+
+    var statefilters = $filter('filter')($scope.active_filters,
+                                         {type: "state"});
+    if (statefilters.length > 0) {
+      matches = false;
+      angular.forEach(statefilters, function(v) {
+        matches = matches || state.toLowerCase().indexOf(v.value.toLowerCase()) != -1;
+      });
+    }
+    if (!matches) return matches;
+
+    return (new Date(lastmod) > new Date($scope.active_filters[$scope.active_filters.length - 1].value));
+  };
+
+  $scope.matches_filter = function(job) {
+    var res = check_filter(job);
+    if (job.shown !== undefined) {
+      if (job.shown != res) {
+        $scope.jobs_shown += res ? 1 : -1;
+      }
+    } else if (res) {
+      $scope.jobs_shown += 1;
+    }
+    job.shown = res;
+    return res;
+  };
+
+  $scope.open = function() {
+    var modalInstance = $modal.open({
+      templateUrl: 'submitModal.html',
+      controller: ModalInstanceCtrl,
+      backdrop: "static",
+    });
+
+    modalInstance.result.then(function(result) {
+      alertService.add('success', "Submitted commit to queue as " + result,
+                       5000);
+    }, function(failure) {
+      alertService.add('danger', "Failed to submit commit: " + failure,
+                       5000);
+    });
+  }
+
+  $scope.abort = function($event, key) {
+    $http({
+      method: "POST",
+      url: "abort",
+      data: 'key=' + encodeURIComponent(key),
+      headers: {'Content-Type': 'application/x-www-form-urlencoded'}
+    }).then(function(data) {
+      console.log("Aborting " + key);
+      alertService.add('warning',
+                       "Aborted request " + key,
+                       5000);
+      $event.target.ngShow = false;
+    }, function(failure) {
+      alertService.add('danger',
+                       'Failed to abort request ' + key + ': ' +
+                       unescape(failure.data.fail), 5000);
+    });
+  };
+
+  $scope.resubmitting = false;
+  $scope.resubmit = function($event, key) {
+    console.log(angular.element($event.target).children());
+    $scope.resubmitting = key;
+    $http({
+      method: "POST",
+      url: "submit",
+      data: 'resubmitKey=' + encodeURIComponent(key),
+      headers: {'Content-Type': 'application/x-www-form-urlencoded'}
+    }).then(function(data) {
+      console.log("Resubmitted " + key + ": " + data);
+      alertService.add('success',
+                       "Resubmitted request " + key + " to the queue",
+                       5000);
+      $event.target.ngShow = false;
+      $scope.resubmitting = false;
+    }, function(failure) {
+      $scope.resubmitting = false;
+      alertService.add('danger',
+                       'Failed to resubmit request ' + key +
+                       ' to the queue: ' + failure.data.fail, 5000);
+    });
+  };
+
+  poll();
+}
+
+var ModalInstanceCtrl = function($scope, $modalInstance, $http, $timeout,
+                                 asyncData, alertService) {
+  $scope.info = {
+    username: "",
+    target: "",
+    commitdiff: "",
+    buglist: "",
+    stages: [],
+    ws_reset: [],
+    ws_disable: [],
+  };
+  $scope.stages = [];
+  $scope.workspaces = [];
+  $scope.stage_properties = {};
+  $scope.targets = [];
+  asyncData.getData("list?arg=target", function(data) {
+    return data.target;
+  }).then(function(targets) {
+    $scope.targets = targets;
+  });
+  $scope.$watch("info.target", function() {
+    if ($scope.fetching_targets) {
+      $timeout.cancel($scope.fetching_targets);
+    }
+    $scope.fetching_targets = $timeout(function() {
+      var tgt = $scope.info.target;
+      asyncData.getData("stages?target=" + encodeURIComponent(tgt))
+      .then(function(data) {
+        $scope.stages = data.stages;
+        $scope.stage_properties = {};
+        $scope.info.stages = [];
+        angular.forEach(data.stages, function(v) {
+          $scope.info.stages.push(v.name);
+          $scope.stage_properties[v.name] = v;
+        });
+        console.log($scope.stage_properties);
+        $scope.fetching_targets = false;
+        $scope.workspaces = data.workspaces;
+      }, function() {
+        console.log("It appears " + tgt + " isn't a valid target");
+        $scope.stages = [];
+        $scope.stage_properties = {};
+        $scope.info.stages = [];
+        $scope.fetching_targets = false;
+      });
+    }, 500);
+  });
+
+  function addStage(s) {
+    if ($scope.info.stages.indexOf(s) != -1) return;
+    console.log("Adding stage " + s);
+    if (!$scope.stage_properties[s]) {
+      alertService.add("danger", "Invalid stage: " + s);
+      return;
+    }
+    angular.forEach($scope.stage_properties[s].deps, function(d) {
+      addStage(d);
+    });
+    $scope.info.stages.push(s);
+  }
+  function removeStage(s) {
+    var idx = $scope.info.stages.indexOf(s);
+    console.log("Removing stage " + s + " from " + idx);
+    if (idx == -1) return;
+    $scope.info.stages.splice(idx, 1);
+    angular.forEach($scope.stage_properties, function(o) {
+      console.log(o);
+      if (o.deps.indexOf(s) != -1) {
+        removeStage(o.name);
+      }
+    });
+  }
+
+  $scope.updateStages = function(stage) {
+    stage = stage.getAttribute('kermit-name');
+    console.log("Updating stage: " + stage);
+    var idx = $scope.info.stages.indexOf(stage);
+    if (idx == -1) {
+      addStage(stage);
+    } else {
+      removeStage(stage);
+    }
+  };
+
+  $scope.updateWorkspaces = function(button) {
+    workspace = button.getAttribute('kermit-name');
+    if ($scope.info.ws_reset.indexOf(workspace) != -1) {
+      $scope.info.ws_reset.splice($scope.info.ws_reset.indexOf(workspace), 1);
+      $scope.info.ws_disable.push(workspace);
+    } else if ($scope.info.ws_disable.indexOf(workspace) != -1) {
+      $scope.info.ws_disable.splice($scope.info.ws_disable.indexOf(workspace), 1);
+    } else {
+      $scope.info.ws_reset.push(workspace);
+    }
+  };
+
+  $scope.workspaceTooltip = function(workspace) {
+    if ($scope.info.ws_reset.indexOf(workspace) != -1) {
+      return "Reset workspace before using it";
+    } else if ($scope.info.ws_disable.indexOf(workspace) != -1) {
+      return "Don't use this shared workspace";
+    } else {
+      return "Use this workspace as configured";
+    }
+  };
+
+  $scope.ok = function() {
+    $http({
+      method: "POST",
+      url: "submit",
+      transformRequest: function(obj) {
+        var str = [];
+        angular.forEach(obj, function(v, k) {
+          str.push(encodeURIComponent(k) + "=" + encodeURIComponent(v));
+        });
+        console.log(str.join('&'));
+        return str.join("&");
+      },
+      data: $scope.info,
+      headers: {'Content-Type': 'application/x-www-form-urlencoded'}
+    }).then(function(data) {
+      console.log(data);
+      $modalInstance.close(data.data.success);
+    }, function(data) {
+      $modalInstance.dismiss(data.data.fail);
+    });
+  };
+
+  $scope.cancel = function() {
+    $modalInstance.dismiss('cancelled by you');
+  };
+};

=== added file 'resources/kermit/response.json'
--- resources/kermit/response.json	1970-01-01 00:00:00 +0000
+++ resources/kermit/response.json	2014-09-01 20:21:07 +0000
@@ -0,0 +1,5 @@
+{% if success %}
+{"success": {{ success }}}
+{% else %}
+{"fail": "{{ fail|escape }}"}
+{% endif %}

=== added file 'resources/kermit/services.js'
--- resources/kermit/services.js	1970-01-01 00:00:00 +0000
+++ resources/kermit/services.js	2014-09-01 20:21:07 +0000
@@ -0,0 +1,39 @@
+
+angular.module("filterFocus", []).directive("filterFocus", function($timeout) {
+  return {
+    link: function(scope, element, attrs) {
+      scope.$watch(attrs.filterFocus, function(value) {
+        if (value) {
+          $timeout(function() {
+            element[0].focus();
+            scope.$eval(attrs.filterFocus + " = false");
+          }, 500);
+        }
+      });
+    }
+  };
+});
+
+angular.module("asyncData", [])
+    .service("asyncData", ['$rootScope', '$q', '$http',
+  function($rootScope, $q, $http) {
+    var asyncData = function(args) {
+      var convertData = args['convertData'] || function(data) { return data; };
+      var deferred = $q.defer();
+
+      $http(args).then(function(data) {
+        deferred.resolve(convertData(data.data));
+        if (!$rootScope.$$phase) $rootScope.$apply();
+      }, function(data) {
+        deferred.reject(data);
+      });
+
+      return deferred.promise;
+    };
+
+    asyncData.getData = function(url, convertData) {
+      return asyncData({method: 'GET', url: url, convertData: convertData});
+    };
+
+    return asyncData;
+  }]);

=== modified file 'src/endroid/database.py'
--- src/endroid/database.py	2014-07-28 15:50:00 +0000
+++ src/endroid/database.py	2014-09-01 20:21:07 +0000
@@ -3,28 +3,174 @@
 # Copyright 2012, Ensoft Ltd.
 # Created by Jonathan Millican
 # -----------------------------------------
+
 import sqlite3
 import os.path
-
-
-# Export constants for system column names
-EndroidUniqueID = '_endroid_unique_id'
+import logging
+
+__all__ = (
+    'Value', 'LIKE',
+    'Field',
+    'Database',
+    'EndroidUniqueID', 'EndroidLastModified',
+)
 
 
 class TableRow(dict):
-    """A regular dict, plus a system 'id' attribute."""
+    """
+    A dict with special properties for access the magic fields of the row.
+
+    These should not be instantiated directly, but are instead returned from
+    "fetch" operations on tables.
+
+    """
     __slots__ = ()
 
     def __init__(self, *args, **kwargs):
         super(TableRow, self).__init__(*args, **kwargs)
-        if not EndroidUniqueID in self:
-            raise ValueError("Cannot create table row from table with no {0} "
-                             "column!".format(EndroidUniqueID))
 
     @property
     def id(self):
         return self[EndroidUniqueID]
 
+    @property
+    def last_modified(self):
+        return self[EndroidLastModified]
+
+
+class Value(object):
+    """
+    Wrapper for a value to be matched in a condition.
+
+    'cmp' must be one of the comparison operators supported by SQL.
+
+    """
+    def __init__(self, val, cmp='='):
+        self._value = str(val)
+        self._cmp = str(cmp)
+
+    def __str__(self):
+        return self._value
+
+    def __or__(self, other):
+        return OR(self, other)
+
+    def to_condition(self, field):
+        return Field.as_str(field) + self._cmp + "?"
+
+    def values(self):
+        return (self._value,)
+
+    @classmethod
+    def get_values(cls, obj):
+        if isinstance(obj, cls):
+            return obj.values()
+        else:
+            return (str(obj),)
+
+    @classmethod
+    def as_condition(cls, obj, field):
+        if isinstance(obj, cls):
+            return obj.to_condition(field)
+        else:
+            return Value.as_condition(Value(obj), field)
+
+
+class LIKE(Value):
+    "Value subclass for handling SQl 'LIKE' comparisons."
+    def __init__(self, val):
+        super(LIKE, self).__init__("%{}%".format(val), "LIKE")
+
+
+class OR(Value):
+    def __init__(self, *vals):
+        self._vals = vals
+
+    def to_condition(self, field):
+        return " OR ".join(Value.as_condition(v, field) for v in self._vals)
+
+    def values(self):
+        vals = []
+        for v in self._vals:
+            vals.extend(Value.get_values(v))
+        return tuple(vals)
+
+
+class Field(object):
+    """
+    Class representing a Field of a Table.
+
+    These need usually only be instantiated when a table is first being
+    created. Note that raw strings can be used as fields in create calls,
+    and they simply get the default type.
+
+    """
+    def __init__(self, name, type=None, default=None):
+        self._name = Field.as_str(name)
+        self._type = type
+        self._default = default
+
+    def __str__(self):
+        return self._name
+
+    @classmethod
+    def as_create_str(cls, obj):
+        if isinstance(obj, cls):
+            return "{} {} {}".format(obj._name, obj._type if obj._type else "",
+                             "default " + obj._default if obj._default else "")
+        else:
+            return Field.as_create_str(Field(obj))
+
+    @classmethod
+    def as_str(cls, obj):
+        if isinstance(obj, cls):
+            return obj._name
+        else:
+            return Database._sanitize(obj)
+
+
+class Table(object):
+    """
+    Wrapper around a table (so that the table name doesn't need to be
+    passed as the first argument all the time).
+
+    """
+    def __init__(self, name, db):
+        self._name = name
+        self._db = db
+
+    def create(self, fields):
+        return self._db.create_table(self._name, fields)
+
+    def exists(self):
+        return self._db.table_exists(self._name)
+
+    def fetch(self, fields, conditions={}, distinct=False, max=False):
+        return self._db.fetch(self._name, fields, conditions, distinct, max)
+
+    def count(self, conditions):
+        return self._db.count(self._name, conditions)
+
+    def insert(self, fields):
+        return self._db.insert(self._name, fields)
+
+    def delete(self, conditions={}):
+        return self._db.delete(self._name, conditions)
+
+    def update(self, fields, conditions):
+        return self._db.update(self._name, fields, conditions)
+
+    def empty(self):
+        return self._db.empty_table(self._name)
+
+    def drop(self):
+        return self._db.delete_table(self._name)
+
+    @property
+    def name(self):
+        return self._db._tName(self._name)
+
+
 class Database(object):
     """
     Wrapper round an sqlite3 Database.
@@ -62,32 +208,48 @@
     
     @staticmethod
     def _stringFromFieldNames(fields):
-        return ", ".join(Database._sanitize(f) for f in fields)
+        return ", ".join(Field.as_str(f) for f in fields)
 
     @staticmethod
     def _stringFromListItems(list):
-        return ", ".join(Database._sanitize(l) for l in list)
+        return ", ".join(Field.as_str(l) for l in list)
     
     @staticmethod
     def _tupleFromFieldValues(fields):
-        return tuple(fields.values())
+        vals = []
+        for v in fields.values():
+            vals.extend(Value.get_values(v))
+        return tuple(vals)
     
     @staticmethod
     def _buildConditions(conditions):
-        return " and ".join(Database._sanitize(c) + "=?" for c in conditions) or "1"
-                    
+        return " and ".join(Value.as_condition(v, f)
+                            for f, v in conditions.items()) or "1"
+
+    def table(self, name):
+        return Table(name, self)
+
     def create_table(self, name, fields):
         """
         Create a new table in the database called 'name' and containing fields 
         'fields' (an iterable of strings giving field titles).
 
         """
-        if any(f.startswith('_endroid') for f in fields):
-            raise ValueError("An attempt was made to create a table with system-reserved column-name (prefix '_endroid').")
+        if any(str(f).startswith('_endroid') for f in fields):
+            raise ValueError("An attempt was made to create a table with "
+                             "system-reserved column-name (prefix '_endroid').")
         n = Database._sanitize(self._tName(name))
-        fields_string = ', '.join(['{0} INTEGER PRIMARY KEY AUTOINCREMENT'.format(EndroidUniqueID)] + [Database._sanitize(i) for i in fields])
+        fields_string = ', '.join(map(Field.as_create_str,
+                                  [EndroidUniqueID] + list(fields)))
         query = 'CREATE TABLE {0} ({1});'.format(n, fields_string)
         Database.raw(query)
+
+        if EndroidLastModified in fields:
+            Database.raw("CREATE TRIGGER {3}_last_modified AFTER UPDATE ON {0} "
+                         "BEGIN UPDATE {0} SET {1} = DATETIME('now') "
+                         "WHERE {2} = new.{2}; END;"
+                         .format(n, EndroidLastModified, EndroidUniqueID,
+                                 self._tName(name)))
     
     def table_exists(self, name):
         """Check to see if a table called 'name' exists in the database."""
@@ -114,7 +276,7 @@
         Database.raw(query, tup)
         return Database.cursor.lastrowid
     
-    def fetch(self, name, fields, conditions={}):
+    def fetch(self, name, fields, conditions={}, distinct=False, max=False):
         """
         Get data from the table 'name'. 
 
@@ -122,30 +284,35 @@
         dictionary for each row which satisfies a condition in conditions.
 
         Conditions is a dictionary mapping field names to values. A result
-        will only be returned from a row if its values match those in conditions.
+        will only be returned from a row if its values match those in conditions
 
-        E.g.: conditions = {'user' : JoeBloggs} 
+        E.g.: conditions = {'user': 'JoeBloggs'} 
         will match only fields in rows which have JoeBloggs in the 'user' field.
 
         """
         n = Database._sanitize(self._tName(name))
-        fields = list(fields) + [EndroidUniqueID]
-        query = "SELECT {0} FROM {1} WHERE ({2});".format(
-            Database._stringFromListItems(fields), n,
-            Database._buildConditions(conditions))
+        if not distinct and not max:
+            fields = list(fields) + [EndroidUniqueID]
+        if max:
+            cols = "MAX({})".format(Database._stringFromListItems(fields))
+        else:
+            cols = Database._stringFromListItems(fields)
+        query = "SELECT {3}{0} FROM {1} WHERE ({2});".format(
+            cols, n, Database._buildConditions(conditions),
+            "DISTINCT " if distinct else "")
         Database.raw(query, Database._tupleFromFieldValues(conditions))
         c = Database.cursor.fetchall()
         rows = [TableRow(dict(zip(fields, item))) for item in c]
         return rows
 
     def count(self, name, conditions):
-        """Return the number of rows in table 'name' which satisfy conditions."""
+        "Return the number of rows in table 'name' which satisfy conditions."
         n = Database._sanitize(self._tName(name))
         query = "SELECT COUNT(*) FROM {0} WHERE ({1});".format(n, Database._buildConditions(conditions))
         r = Database.raw(query, Database._tupleFromFieldValues(conditions)).fetchall()
         return r[0][0]
 
-    def delete(self, name, conditions):
+    def delete(self, name, conditions={}):
         """Delete rows from table 'name' which satisfy conditions."""
         n = Database._sanitize(self._tName(name))
         query = "DELETE FROM {0} WHERE ({1});".format(n, Database._buildConditions(conditions))
@@ -168,18 +335,23 @@
 
     def empty_table(self, name):
         """Remove all rows from table 'name'."""
-        n = Database._sanitize(self._tName(name))
-        query = "DELETE FROM {0} WHERE 1;".format(n)
-        Database.raw(query)
+        return self.delete(name)
     
     def delete_table(self, name):
         """Delete table 'name'."""
         n = Database._sanitize(self._tName(name))
         query = "DROP TABLE {0};".format(n)
         Database.raw(query)
-    
+
     @staticmethod
     def raw(command, params=()):
+        logging.debug("SQL command {} params {}".format(command, params))
         p = Database.cursor.execute(command, params)
         Database.connection.commit()
         return p
+
+# Export constants for system column names
+EndroidUniqueID = Field('_endroid_unique_id',
+                        type="INTEGER PRIMARY KEY AUTOINCREMENT")
+EndroidLastModified = Field('_endroid_last_modified', type="DATETIME",
+                            default="current_timestamp")

=== modified file 'src/endroid/pluginmanager.py'
--- src/endroid/pluginmanager.py	2014-08-11 15:30:01 +0000
+++ src/endroid/pluginmanager.py	2014-09-01 20:21:07 +0000
@@ -304,12 +304,12 @@
         try:
             __import__(modname)
         except ImportError as i:
-            logging.error(i)
+            logging.exception(i)
             logging.error("**Could Not Import Plugin \"" + modname
                           + "\". Check That It Exists In Your PYTHONPATH.")
             return
         except Exception as e:
-            logging.error(e)
+            logging.exception(e)
             logging.error("**Failed to import plugin {}".format(modname))
             return
         else:

=== modified file 'src/endroid/plugins/httpinterface.py'
--- src/endroid/plugins/httpinterface.py	2014-04-05 11:51:30 +0000
+++ src/endroid/plugins/httpinterface.py	2014-09-01 20:21:07 +0000
@@ -189,7 +189,7 @@
                                       [BasicCredentialFactory("EnDroid")])
 
     def endroid_init(self, pluginmanager, port, interface, media_dir,
-                     template_dir, credplugins):
+                     template_dir, credplugins, baseurl):
         def _get_cred(plugin):
             """
             Get a credential checker for one entry in the credplugins list.
@@ -198,14 +198,13 @@
             try:
                 cred = plugin.http_cred_checker()
             except Exception as e:
-                logging.exception(e)
                 try:
                     cred = pluginmanager.get(plugin).http_cred_checker()
                 except Exception as e:
                     logging.exception(e)
                     cred = None
             return cred
-        self._creds = [_get_cred(p) for p in credplugins]
+        self._creds = [_get_cred(p) for p in credplugins if p]
         self._root = IndexPage(self)
         self._media = File(media_dir)
         self._root.putChild("_media", self._media)
@@ -216,6 +215,8 @@
             return time.strftime(fmt, time.localtime(val))
         self.jinja.filters['datetime'] = datetime_filter
 
+        self.baseurl = baseurl
+
         factory = Site(self._root)
         logging.info("Starting web server on {}:{}; static files in {}"
                      .format(interface, port, media_dir))
@@ -303,6 +304,10 @@
         """
         return InMemoryUsernamePasswordDatabaseDontUse(endroid='password')
 
+    @property
+    def baseurl(self):
+        return HTTPInterface._singleton.baseurl
+
     def endroid_init(self):
         """
         Initialises the singleton object on first call only.
@@ -315,7 +320,9 @@
             media_dir = self.vars.get("media_dir", DEFAULT_MEDIA_DIR)
             templ_dir = self.vars.get("template_dir", DEFAULT_TEMPL_DIR)
             credplugins = self.vars.get("credential_plugins", [self])
+            baseurl = self.vars.get("baseurl", "http://localhost/")
             HTTPInterface._singleton.endroid_init(self.plugins, port,
                                                   interface, media_dir,
-                                                  templ_dir, credplugins)
+                                                  templ_dir, credplugins,
+                                                  baseurl)
             HTTPInterface.enInited = True

=== added directory 'src/endroid/plugins/kermit'
=== added file 'src/endroid/plugins/kermit/__init__.py'
--- src/endroid/plugins/kermit/__init__.py	1970-01-01 00:00:00 +0000
+++ src/endroid/plugins/kermit/__init__.py	2014-09-01 20:21:07 +0000
@@ -0,0 +1,309 @@
+# -----------------------------------------------------------------------------
+# EnDroid - Remote notification plugin
+# Copyright 2012, Ensoft Ltd
+# -----------------------------------------------------------------------------
+
+import os
+import re
+import json
+import time
+import shlex
+import logging
+import cfgparser
+
+from Crypto.PublicKey import RSA
+from twisted.conch.ssh import keys
+from twisted.internet import reactor, defer, threads, task
+from twisted.internet.protocol import ProcessProtocol
+
+from endroid.database import Database, EndroidUniqueID, EndroidLastModified
+from endroid.usermanagement import JID
+from endroid.plugins.command import CommandPlugin, command
+
+from .web import KermitRoot
+from . import ssh
+
+
+class Kermit(CommandPlugin):
+    dependencies = ('endroid.plugins.httpinterface',)
+    help = "Automatically commit patches without having to do the legwork."
+    DB_TABLE = "Queue"
+    KEY_TABLE = "Keys"
+
+    def endroid_init(self):
+        self.http = self.get('endroid.plugins.httpinterface')
+        self.http.register_resource(self, KermitRoot(self))
+
+        self.config_dir = self.vars.get("config_dir")
+        self.trigger_path = self.vars.get("trigger_path", "kermit_trigger")
+        self.qr_path = self.vars.get("qr_path", "kermit_queue_runner")
+
+        self.db = self.database.table(self.DB_TABLE)
+        self.keydb = self.database.table(self.KEY_TABLE)
+
+        if not self.db.exists():
+            self.db.create(("user", "target", "status", EndroidLastModified))
+        if not self.keydb.exists():
+            self.keydb.create(("user", "keystr"))
+
+        self.waiters = []
+
+
+    def notify_waiters(self):
+        for querier, waiter in self.waiters:
+            reactor.callLater(1, querier.respond, waiter)
+        del self.waiters[:]
+
+
+    def read_config(self, target=None):
+        files = [os.path.join(self.config_dir, "kermit.cfg")]
+        if target:
+            files.append(os.path.join(self.config_dir, target + ".cfg"))
+        c = cfgparser.CfgParser()
+        c.read(files)
+        return c
+
+
+    # Key Management
+
+    def _generate_key(self, username):
+        key = RSA.generate(2048)
+        return keys.Key.fromString(key.exportKey("PEM"))
+
+    def generate_key(self, username):
+        return threads.deferToThread(self._generate_key, username)
+
+    def add_key(self, username, keystr):
+        key = keys.Key.fromString(keystr)
+        if key.isPublic():
+            raise ValueError("Uploaded key must be a private key")
+
+        id = self.keydb.insert({
+          "user": username,
+          "keystr": key.toString("OPENSSH"),
+        })
+        return {
+            'idx': id,
+            'type': key.public().type(),
+            'keystr': key.public().toString("OPENSSH"),
+        }
+
+
+    def remove_key(self, username, idx):
+        self.keydb.delete({"user": username, EndroidUniqueID: idx})
+
+
+    def get_private_keys(self, username):
+        rows = self.keydb.fetch(("keystr",), {"user": username})
+        return [keys.Key.fromString(k['keystr']) for k in rows]
+
+
+    def list_keys(self, username):
+        rows = self.keydb.fetch(("keystr",), {"user": username})
+        ks = ((k.id, keys.Key.fromString(k['keystr']).public()) for k in rows)
+        return [{'idx': i, 'type': k.type(), 'keystr': k.toString("OPENSSH")}
+                for i, k in ks]
+
+
+    @staticmethod
+    def _parse(arg):
+        parts = shlex.split(arg)
+
+        if len(parts) != 5:
+            raise ValueError("Invalid commit command")
+        if parts[1].lower() != 'to' or parts[3].lower() != 'against':
+            raise ValueError("Malformed commit command")
+
+        return parts[0::2]
+
+
+    def abort_commit(self, deferred, key):
+        row = self.db.fetch(("target", "user", "status"),
+                             {EndroidUniqueID: key})
+        if not row:
+            raise ValueError("Unknown queue item ({})".format(key))
+        target = row[0]['target']
+        user = row[0]['user']
+        status = json.loads(row[0]['status'])
+
+        config = self.read_config(target)
+        config.set("DEFAULT", "target", target)
+        config.set("DEFAULT", "username", user)
+        config.set("DEFAULT", "key", "kermit-{}".format(key))
+
+        def _reset_status(_):
+            status['currstatus']['running'] = []
+            status['currstatus']['failed'] = []
+            status['currstatus']['failed_at'] = {}
+            status['aborted_at'] = time.time()
+            self.db.update({
+                "status": json.dumps(status),
+            }, {EndroidUniqueID: key})
+            self.notify_waiters()
+            return key
+
+        d = ssh.spawn_trigger(config.get(target, "server", default="localhost"),
+                              config.getlist(target, "fingerprints",
+                                             default=[]),
+                              user, self.get_private_keys(user),
+                              "-a kermit-{}".format(key),
+                              config)
+
+        d.addCallback(_reset_status)
+        d.chainDeferred(deferred)
+
+
+    def resubmit_commit(self, deferred, key):
+        row = self.db.fetch(("status",), {EndroidUniqueID: key})
+        if not row:
+            raise ValueError("Unknown queue item ({})".format(key))
+        args = json.loads(row[0]['status'])['inputargs']
+
+        username = args['username']
+        commitdiff = args['commitdiff']
+        target = args['target']
+        buglist = args['buglist']
+        commitmsg = args['commitmsg']
+        stages = args.get('stages', '')
+        ws_disable = args.get('ws_disable', '')
+        ws_reset = args.get('ws_reset', '')
+
+        self.submit_commit(deferred, username, commitdiff, target, buglist,
+                           commitmsg, stages, key=key, ws_disable=ws_disable,
+                           ws_reset=ws_reset)
+
+
+    def submit_commit(self, deferred, user, commitdiff, target, buglist,
+                      commitmsg='', stages='', key=None, ws_disable='',
+                      ws_reset=''):
+        if not os.path.exists(os.path.join(self.config_dir, target + '.cfg')):
+            deferred.errback(OSError("No configuration file for target"))
+            return
+        config = self.read_config(target)
+        config.set("DEFAULT", "commitdiff", commitdiff)
+        config.set("DEFAULT", "target", target)
+        config.set("DEFAULT", "buglist", buglist)
+        config.set("DEFAULT", "commitmsg", commitmsg)
+        config.set("DEFAULT", "username", user)
+        config.set("DEFAULT", "stages", stages)
+        config.set("DEFAULT", "ws_disable", ws_disable)
+        config.set("DEFAULT", "ws_reset", ws_reset)
+        config.set("DEFAULT", "qr_path", self.qr_path)
+        config.set("DEFAULT", "trigger_path", self.trigger_path)
+        config.set("DEFAULT", "kermit_baseurl", self.http.baseurl)
+
+        def _reset_status(_):
+            rows = self.db.fetch(("status",), {EndroidUniqueID: key})
+            if rows:
+                status = json.loads(rows[0]['status'])
+                status['currstatus']['running'] = []
+                status['currstatus']['failed'] = []
+                status['currstatus']['failed_at'] = {}
+                status['aborted_at'] = 0
+                self.db.update({
+                    "status": json.dumps(status),
+                }, {EndroidUniqueID: key})
+                self.notify_waiters()
+            return _
+
+        deferred.addCallback(_reset_status)
+
+        args = {
+            'username': user,
+            'commitdiff': commitdiff,
+            'target': target,
+            'buglist': buglist,
+            'commitmsg': commitmsg,
+            'stages': stages,
+            'ws_disable': ws_disable,
+            'ws_reset': ws_reset,
+        }
+
+        if key:
+            rowid = key
+        else:
+            rowid = self.db.insert({
+                "user": user,
+                "target": target,
+                "status": json.dumps({
+                    'inputargs': args,
+                    'currstatus': {},
+                    'aborted_at': 0,
+                }),
+            })
+            def _cleanup(fail):
+                self.db.delete({EndroidUniqueID: rowid})
+                return fail
+            deferred.addErrback(_cleanup)
+        config.set("DEFAULT", "key", "kermit-{}".format(rowid))
+
+        workspaces = config.getlist(target, "workspaces")
+        server = config.get(target, "server", default="localhost")
+        fprints = config.getlist(target, "fingerprints", default=[])
+
+        ds = []
+        for ws in workspaces:
+            section = config.section(ws, category="workspace")
+            shuser = section.get("username", default=user)
+
+            logging.info("Spawning workspace process: {}@{}"
+                         .format(shuser, server))
+            d = ssh.spawn_trigger(server, fprints,
+                                  shuser, self.get_private_keys(shuser),
+                                  "-w", config)
+
+            ds.append(d)
+
+        dl = defer.DeferredList(ds, fireOnOneErrback=True, consumeErrors=True)
+        # Sleep a little bit to give the shared workspaces time to start
+        # listening
+        def _get_subfailure(fail):
+            fail.trap(defer.FirstError)
+            return fail.value.subFailure
+        dl.addCallback(lambda _: task.deferLater(reactor, 2, lambda: _))
+        dl.addCallbacks(lambda _: ssh.spawn_trigger(server, fprints, user,
+                                                    self.get_private_keys(user),
+                                                    "", config),
+                        _get_subfailure)
+        dl.chainDeferred(deferred)
+    
+
+    @command(helphint="<patch> to <target> against <buglist>")
+    def commit(self, msg, arg):
+        commitdiff, target, buglist = self._parse(arg)
+
+        d = defer.Deferred()
+        d.addCallbacks(lambda k: msg.reply("Commit started as {}".format(k)),
+                       lambda f: msg.reply("Failed to start commit: {}"
+                                           .format(f)))
+
+        self.submit_commit(d, JID(msg.sender_full).user, commitdiff,
+                           target, buglist)
+
+
+    @command(helphint="<commit-id>")
+    def abort(self, msg, arg):
+        d = defer.Deferred()
+        d.addCallbacks(lambda k: msg.reply("Commit {} abort".format(k)),
+                       lambda f: msg.reply("Failed to abort commit {}: {}"
+                                           .format(arg, f)))
+        self.abort_commit(d, int(arg))
+
+
+    @command(helphint="<commit-id>")
+    def resubmit(self, msg, arg):
+        d = defer.Deferred()
+        d.addCallbacks(lambda k: msg.reply("Commit {} resubmitted".format(k)),
+                       lambda f: msg.reply("Failed to resubmit commit {}: {}"
+                                           .format(arg, f)))
+        self.resubmit_commit(d, int(arg))
+
+
+    @command(synonyms=(('clear', 'commit'),), hidden=True)
+    def clear_commits(self, msg, arg):
+        if arg:
+            self.db.delete({EndroidUniqueID: int(arg)})
+        else:
+            self.db.delete()
+        self.notify_waiters()
+        msg.reply("Done.")

=== added file 'src/endroid/plugins/kermit/ssh.py'
--- src/endroid/plugins/kermit/ssh.py	1970-01-01 00:00:00 +0000
+++ src/endroid/plugins/kermit/ssh.py	2014-09-01 20:21:07 +0000
@@ -0,0 +1,301 @@
+
+import struct
+import logging
+import weakref
+
+from twisted.conch.ssh import (
+    transport, keys, connection, channel, common, userauth, filetransfer,
+)
+from twisted.internet import reactor, defer, protocol, task, error
+
+# Connection classes
+
+class _Transport(transport.SSHClientTransport):
+
+    def __init__(self, deferred, fingerprints, username, keys):
+        self._deferred = deferred
+        self._fingerprints = fingerprints
+        self._username = username
+        self._keys = keys
+        self.connected = False
+
+    def verifyHostKey(self, key, fingerprint):
+        for f in self._fingerprints:
+            if f.lower() == 'ignore' or f.lower() == fingerprint.lower():
+                return defer.succeed(True)
+        logging.warning("Fingerprint doesn't match: " + fingerprint)
+        d, self._deferred = self._deferred, None
+        err = ValueError("fingerprint ({}) doesn't match config"
+                         .format(fingerprint))
+        d.errback(err)
+        return defer.fail(err)
+
+    def connectionSecure(self):
+        self.connected = True
+        self.requestService(_Auth(str(self._username),
+                                  _Conn(self._deferred,
+                                        self._connection_authed),
+                                  self._keys))
+
+    def _connection_authed(self):
+        self._deferred = None
+
+    def connectionLost(self, reason):
+        self.connected = False
+        logging.warning("Lost connection: " + str(reason))
+        d, self._deferred = self._deferred, None
+        if d is not None:
+            d.errback(ValueError("failed to setup connection for user {}"
+                                 .format(self._username)))
+
+class _Auth(userauth.SSHUserAuthClient):
+
+    def __init__(self, username, conn, keys):
+        userauth.SSHUserAuthClient.__init__(self, username, conn)
+        self._lastkey = None
+        self._keys = keys
+
+    def getPassword(self, prompt=None):
+        return
+
+    def getPublicKey(self):
+        if not self._keys:
+            self._lastkey = None
+            return None
+        self._lastkey = self._keys.pop(0)
+        return self._lastkey.public()
+
+    def getPrivateKey(self):
+        return defer.succeed(self._lastkey)
+
+class _Conn(connection.SSHConnection):
+
+    def __init__(self, deferred, authfn):
+        connection.SSHConnection.__init__(self)
+        self._deferred = deferred
+        self._authfn = authfn
+        self._timeout = None
+
+    def serviceStarted(self):
+        connection.SSHConnection.serviceStarted(self)
+        self._authfn()
+        d, self._deferred = self._deferred, None
+        d.callback(self)
+
+    def serviceStopped(self):
+        connection.SSHConnection.serviceStopped(self)
+
+    def openChannel(self, channel, extra=''):
+        if self._timeout:
+            try:
+                self._timeout.cancel()
+            except error.AlreadyCalled as e:
+                logging.error("Cancelling already called timeout. Currently connected? {}".format(self.transport.connected))
+            self._timeout = None
+        connection.SSHConnection.openChannel(self, channel, extra=extra)
+
+    def channelClosed(self, channel):
+        connection.SSHConnection.channelClosed(self, channel)
+        if len(self.channels) == 0:
+            self._timeout = reactor.callLater(600, self._disconnect)
+
+    def _disconnect(self):
+        logging.info("Connection for {} timed out"
+                     .format(self.transport._username))
+        self.transport.sendDisconnect(transport.DISCONNECT_BY_APPLICATION,
+                                      "connection keepalive timeout reached")
+
+class _Factory(protocol.ClientFactory):
+
+    def __init__(self, deferred, fingerprints, username, keys):
+        self._deferred = deferred
+        self._fingerprints = fingerprints
+        self._username = username
+        self._keys = keys
+
+    def buildProtocol(self, addr):
+        trans = _Transport(self._deferred, self._fingerprints,
+                           self._username, self._keys)
+        self._deferred = None
+        return trans
+
+    def clientConnectionFailed(self, connector, reason):
+        d, self._deferred = self._deferred, None
+        d.errback(reason)
+
+# Trigger command classes
+
+class TriggerError(Exception):
+    def __init__(self, command, retcode, sig=None):
+        self.command = command
+        self.retcode = retcode
+        self.sig = sig
+        super(TriggerError, self).__init__((command, retcode, sig))
+
+    def __str__(self):
+        return "process '{}' failed: {}{}".format(self.command, self.retcode,
+                                                  " {}".format(self.sig)
+                                                      if self.sig else "")
+
+class _TriggerChan(channel.SSHChannel):
+    name = 'session'
+
+    def __init__(self, deferred, args, config, conn):
+        channel.SSHChannel.__init__(self, conn=conn)
+        self._deferred = deferred
+        self._config = config
+        self._command = config.get("DEFAULT", "trigger_path")
+        if args:
+            self._command += " " + args
+        self._retcode = None
+        self._sig = None
+
+    def channelOpen(self, data):
+        d = self.conn.sendRequest(self, 'exec', common.NS(self._command),
+                                  wantReply=True)
+        d.addCallbacks(self._spawned, self._failed)
+
+    def _spawned(self, _):
+        self._config.write(self)
+        self.conn.sendEOF(self)
+
+    def _failed(self, failure):
+        d, self._deferred = self._deferred, None
+        d.errback(failure)
+
+    def request_exit_status(self, data):
+        self._retcode = struct.unpack("!i", data)[0]
+        logging.info("Exit status received: {}".format(self._retcode))
+
+    def request_exit_signal(self, data):
+        strlen = struct.unpack("!i", data[:4])[0]
+        sig, core, msglen = struct.unpack("!{}s?i".format(strlen),
+                                          data[4:strlen+9])
+        msg, langlen = struct.unpack("!{}si".format(msglen),
+                                     data[9+strlen:13+strlen+msglen])
+        lang = struct.unpack("!{}s".format(langlen), data[13+strlen+msglen:])[0]
+        logging.info("Signal {}. Core: {}. Msg: {}. Lang: {}"
+                     .format(sig, core, msg, lang))
+        self._retcode = -1
+        self._sig = sig
+
+    def dataReceived(self, data):
+        logging.info("Received some data: " + repr(data))
+
+    def extReceived(self, dataType, data):
+        if dataType == connection.EXTENDED_DATA_STDERR:
+            logging.info("Stderr data: {}".format(repr(data)))
+        else:
+            logging.warning("Unexpected ext data (type {}): {}"
+                            .format(dataType, repr(data)))
+
+    def closed(self):
+        logging.info("Channel has closed")
+        d, self._deferred = self._deferred, None
+        if self._retcode == 0:
+            d.callback(None)
+        else:
+            d.errback(TriggerError(self._command, self._retcode, self._sig))
+
+# Log file reading classes
+
+class _LogfileChan(channel.SSHChannel):
+    name = "session"
+
+    def __init__(self, deferred, conn):
+        channel.SSHChannel.__init__(self, conn=conn)
+        self._deferred = deferred
+        self._client = None
+
+    def channelOpen(self, data):
+        d = self.conn.sendRequest(self, 'subsystem', common.NS('sftp'),
+                                  wantReply=True)
+        d.addCallbacks(self._connected, self._failed)
+
+    def _connected(self, _):
+        self._client = filetransfer.FileTransferClient()
+        self._client.makeConnection(self)
+        self.dataReceived = self._client.dataReceived
+        d, self._deferred = self._deferred, None
+        d.callback(self._client)
+
+    def _failed(self, failure):
+        d, self._deferred = self._deferred, None
+        d.errback(failure)
+
+    def closed(self):
+        self._client = None
+
+
+# Public API (and caching var)
+
+_conn_cache = weakref.WeakValueDictionary()
+
+def connect(server, fingerprints, username, keys, port=22):
+    """
+    Connect to username@server:port with keys, and verify server fingerprint.
+
+    This will use a cached connection if one exists for the given server,
+    port, username combo (in this case, it does not reverify the fingerprint,
+    nor use the keys - they are assumed to have been the same or equivalent
+    when the cached connection was set up).
+
+    Returns a Deferred that fires with the SSHConnection instance (on which
+    e.g. openChannel can then be called).
+
+    """
+    if (server, port, username) in _conn_cache:
+        conn = _conn_cache[(server, port, username)]
+        if conn.transport.connected:
+            return defer.succeed(_conn_cache[(server, port, username)])
+
+    d = defer.Deferred()
+    factory = _Factory(d, fingerprints, username, keys)
+    reactor.connectTCP(server, port, factory)
+    def _cache_conn(conn):
+        _conn_cache[(server, port, username)] = conn
+        return conn
+    d.addCallback(_cache_conn)
+    return d
+
+
+def get_logviewer(server, fingerprints, username, keys, port=22):
+    """
+    Gets an sftp subsystem channel to the remote server.
+
+    Returns a Deferred that fires with a FileTransferClient. You should only
+    really call the following on it:
+    - openFile
+    - openDirectory
+    - transport.loseConnection
+
+    The latter of these MUST be called when the connection is no longer
+    required.
+
+    """
+    d = defer.Deferred()
+    d2 = connect(server, fingerprints, username, keys, port=port)
+    d2.addCallback(lambda c: c.openChannel(_LogfileChan(d, conn=c)))
+    d2.addErrback(lambda f: d.errback(f))
+    return d
+
+
+def spawn_trigger(server, fingerprints, username, keys, args, config, port=22):
+    """
+    Spawns the kermit_trigger command on the given server.
+
+    Additional arguments can be passed in 'args'; the CfgParser object
+    containing the config for the commit being affected must be passed in
+    'config'.
+
+    Returns a Deferred that fires (with None) when the process completes, or
+    with the error, which could be a TriggerError if the process returns a
+    non-zero exit code.
+
+    """
+    d = defer.Deferred()
+    d2 = connect(server, fingerprints, username, keys, port=port)
+    d2.addCallback(lambda c:
+                       c.openChannel(_TriggerChan(d, args, config, conn=c)))
+    d2.addErrback(lambda f: d.errback(f))
+    return d

=== added file 'src/endroid/plugins/kermit/web.py'
--- src/endroid/plugins/kermit/web.py	1970-01-01 00:00:00 +0000
+++ src/endroid/plugins/kermit/web.py	2014-09-01 20:21:07 +0000
@@ -0,0 +1,398 @@
+# -----------------------------------------------------------------------------
+# EnDroid - Kermit plugin web interface
+# Copyright 2014, Ensoft Ltd
+# -----------------------------------------------------------------------------
+
+import re
+import json
+import os.path
+import logging
+import datetime
+import cfgparser
+
+from endroid.database import EndroidUniqueID, LIKE, OR, Value, EndroidLastModified
+
+from twisted.web import resource, server, util
+from twisted.python import failure
+from twisted.internet import defer, reactor
+from twisted.conch.ssh import filetransfer
+
+from . import ssh
+
+def _parse_date(dt, fmt="%Y-%m-%dT%H:%M:%S.%fZ"):
+    return datetime.datetime.strptime(dt, fmt)
+
+
+def _parse_key(key):
+    m = re.match("kermit-(\d+)", key)
+    if not m:
+        raise ValueError("Malformed entry index ({})"
+                         .format(request.args["key"][0]))
+    return int(m.group(1))
+
+
+class KermitPage(resource.Resource):
+    isLeaf = True
+
+    def __init__(self, plugin):
+        resource.Resource.__init__(self)
+        self._plugin = plugin
+
+    def _render_success(self, request, key):
+        request.setHeader("Content-Type", "text/json")
+        request.write(self._plugin.http.get_template("kermit/response.json")
+                      .render({
+            "success": json.dumps(key),
+        }).encode('utf-8'))
+        request.finish()
+        return server.NOT_DONE_YET
+
+    def _render_fail(self, request, fail):
+        request.setHeader("Content-Type", "text/json")
+        request.setResponseCode(500)
+        request.write(self._plugin.http.get_template("kermit/response.json")
+                      .render({
+            "fail": str(fail.value),
+        }).encode("utf-8"))
+        request.finish()
+        return server.NOT_DONE_YET
+
+
+class KermitStager(KermitPage):
+    def render_GET(self, request):
+        target = request.args['target'][0]
+        config = self._plugin.read_config(target)
+        request.setHeader("Content-Type", "application/json")
+        if not config.has_section(target):
+            request.setResponseCode(404)
+            return json.dumps({"fail": "no such section {}".format(target)})
+
+        result = []
+        for stage in config.getlist(target, "commands"):
+            section = config.section(stage, category="command")
+            result.append({
+                "name": stage,
+                "deps": section.getlist("dependencies", default=[]),
+            })
+        return json.dumps({
+            "stages": result,
+            "workspaces": config.getlist(target, "workspaces", default=[])
+        })
+
+
+class KermitLister(KermitPage):
+    def render_GET(self, request):
+        field = request.args['arg'][0]
+        rows = self._plugin.db.fetch((field,), distinct=True)
+        request.setHeader("Content-Type", "application/json")
+        return json.dumps({field: [r[field] for r in rows]})
+
+
+class KermitUpdater(KermitPage):
+    def _error(self, request, error):
+        return self._plugin.http.get_template("kermit/response.json").render({
+            'fail': error,
+        }).encode("utf-8")
+
+    def render_POST(self, request):
+        request.setHeader("Content-Type", "application/json")
+        m = re.match("/kermit/update/kermit-(\d+)", request.path)
+        if not m:
+            return self._error(request, "Invalid update URL")
+
+        key = int(m.group(1))
+        data = json.loads(request.content.read())
+
+        row = self._plugin.db.fetch(("status",), {EndroidUniqueID: key})
+        if not row:
+            return self._error(request, "Unknown commit ID: {}".format(key))
+        row = row[0]
+
+        olddata = json.loads(row['status'])
+        olddata['currstatus'] = data
+        self._plugin.db.update({'status': json.dumps(olddata)},
+                               {EndroidUniqueID: key})
+
+        self._plugin.notify_waiters()
+
+        return self._plugin.http.get_template("kermit/response.json").render({
+            'success': key,
+        }).encode('utf-8')
+
+
+class KermitQuerier(KermitPage):
+    def render_POST(self, request):
+        def _cancel(_):
+            logging.info("Cancelling long poll due to client disconnect")
+            self._plugin.waiters.remove((self, request))
+        lastupdate = request.args.get('wait', None)
+        logging.info("Query received. Args are {}".format(request.args))
+        if lastupdate:
+            res = self._plugin.db.fetch((EndroidLastModified,), max=True)
+            logging.info("Latest of all updates was {}".format(res))
+            if res:
+                latest = datetime.datetime.strptime(res[0].last_modified, "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+                logging.info("Waiting for {}. Latest is {}.".format(lastupdate, latest))
+                if latest <= lastupdate[0]:
+                    self._plugin.waiters.append((self, request))
+                    request.notifyFinish().addErrback(_cancel)
+                    return server.NOT_DONE_YET
+            
+        return self.respond(request)
+
+    def respond(self, request):
+        conds = {}
+        if 'username' in request.args:
+            conds['user'] = OR(*(LIKE(u) for u in request.args['username']))
+        if 'target' in request.args:
+            conds['target'] = OR(*(LIKE(u) for u in request.args['target']))
+        if 'lastmod' in request.args:
+            conds[EndroidLastModified] = Value(request.args['lastmod'][0], '<')
+        jobs = self._plugin.db.fetch(("status", EndroidLastModified), conds)
+        request.setHeader("Content-Type", "application/json")
+        res = []
+        for job in jobs:
+            status = json.loads(job['status'])
+            cfg = self._plugin.read_config(status['inputargs']['target'])
+            config = {
+              "commands": {},
+              "workspaces": {},
+            }
+            for stage in status['currstatus'].get('running', []):
+                section = cfg.section(stage, category="command")
+                config['commands'][stage] = dict(section.items())
+            for stage in status['currstatus'].get('failed', []):
+                if cfg.has_section(stage, category='command'):
+                    section = cfg.section(stage, category="command")
+                    config['commands'][stage] = dict(section.items())
+            for stage in status['currstatus'].get('stages', []):
+                section = cfg.section(stage, category="command")
+                config['commands'][stage] = dict(section.items())
+            for ws in cfg.getlist(status['inputargs']['target'], "workspaces"):
+                section = cfg.section(ws, category="workspace")
+                config['workspaces'][ws] = dict(section.items())
+            lastmod = datetime.datetime.strptime(job.last_modified, "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+            res.append({
+                "status": status,
+                "config": config,
+                "key": "kermit-{}".format(job.id),
+                "lastmod": lastmod,
+                "status_string":
+                    "Completed" if status['currstatus'].get('completed_at',
+                                                            False) else
+                    "Failed" if status['currstatus'].get('failed_at', False) or
+                                status['currstatus'].get('check_failed',
+                                                         False) else
+                    "Aborted" if status.get('aborted_at', False) else
+                    "Running" if status['currstatus'].get('running', False) else
+                    "Queued",
+            })
+        latest = self._plugin.db.fetch((EndroidLastModified,), max=True)
+        if latest:
+            latest = datetime.datetime.strptime(latest[0].last_modified, "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        request.write(json.dumps({"last_update": latest, "jobs": res}))
+        request.finish()
+        return server.NOT_DONE_YET
+
+
+class KermitSubmitter(KermitPage):
+    def render_POST(self, request):
+        d = defer.Deferred()
+        d.addCallbacks(lambda k: self._render_success(request, k),
+                       lambda f: self._render_fail(request, f))
+
+        try:
+            if "resubmitKey" in request.args:
+                key = _parse_key(request.args["resubmitKey"][0])
+                self._plugin.resubmit_commit(d, key)
+            else:
+                self._plugin.submit_commit(d,
+                                       request.args['username'][0],
+                                       request.args['commitdiff'][0],
+                                       request.args['target'][0],
+                                       request.args['buglist'][0],
+                                       request.args.get('commitmsg', [''])[0],
+                                       request.args['stages'][0],
+                                       ws_disable=request.args['ws_disable'][0],
+                                       ws_reset=request.args['ws_reset'][0])
+        except Exception as e:
+            logging.exception(e)
+            d.errback(e)
+
+        return server.NOT_DONE_YET
+
+
+class KermitAborter(KermitPage):
+    def render_POST(self, request):
+        d = defer.Deferred()
+        d.addCallbacks(lambda k: self._render_success(request, k),
+                       lambda f: self._render_fail(request, f))
+        try:
+            key = _parse_key(request.args["key"][0])
+            self._plugin.abort_commit(d, key)
+        except Exception as e:
+            logging.exception(e)
+            d.errback(e)
+
+        return server.NOT_DONE_YET
+
+
+class KermitQueue(KermitPage):
+    def render_GET(self, request):
+        items = self._plugin.db.fetch((EndroidUniqueID, "user",
+                                       "target", "status"))
+
+        for item in items:
+            item['key'] = "kermit-{}".format(item.id)
+            item['status'] = json.loads(item['status'])
+            config = self._plugin.read_config(item['target'])
+            item['config'] = config
+
+        return self._plugin.http.get_template("kermit/queue.html").render({
+            "items": items,
+            "username": request.getUser(),
+        }).encode("utf-8")
+
+
+# Key Management
+
+class KermitProfile(KermitPage):
+    def render_GET(self, request):
+        return self._plugin.http.get_template("kermit/profile.html").render({
+            "username": request.getUser(),
+            "keys": self._plugin.list_keys(request.getUser()),
+        }).encode("utf-8")
+
+
+class KermitUploadKey(KermitPage):
+    def render_POST(self, request):
+        try:
+            key = self._plugin.add_key(request.getUser(),
+                                       request.args["privatekey"][0])
+        except Exception as e:
+            return self._render_fail(request, failure.Failure(e))
+        else:
+            return self._render_success(request, key)
+
+
+class KermitGenKey(KermitPage):
+    def render_GET(self, request):
+        def _save_key(key):
+            return self._plugin.add_key(request.getUser(),
+                                        key.toString("OPENSSH"))
+        d = self._plugin.generate_key(request.getUser())
+        d.addCallback(_save_key)
+        d.addCallbacks(lambda k: self._render_success(request, k),
+                       lambda f: self._render_fail(request, f))
+        return server.NOT_DONE_YET
+
+
+class KermitDelKey(KermitPage):
+    def render_GET(self, request):
+        try:
+            self._plugin.remove_key(request.getUser(), request.args['idx'][0])
+        except Exception as e:
+            return self._render_fail(request, failure.Failure(e))
+        else:
+            return self._render_success(request, 'done')
+
+
+# Logfile access
+
+_CHUNK_SIZE = 1024 * 1024
+
+class LogViewer(KermitPage):
+    def render_GET(self, request):
+        self._get_log(request)
+        return server.NOT_DONE_YET
+
+    @defer.inlineCallbacks
+    def _get_log(self, request):
+        key = _parse_key(request.args['key'][0])
+        row = self._plugin.db.fetch(("status",), {EndroidUniqueID: key})
+
+        if not row:
+            self._render_fail(request,
+                        failure.Failure(KeyError("no such commit entry: {}"
+                                           .format(request.args['key'][0]))))
+            return
+        job = row[0]
+
+        status = json.loads(job['status'])
+        target = status['inputargs']['target']
+        cfg = self._plugin.read_config(target)
+        section = cfg.section(target)
+        user = request.getUser()
+
+        # The userdir in the filename is the owner of the operation, not
+        # (necessarily) the person making the request
+        filename = os.path.join(section.get("dir"),
+                                status['inputargs']['username'], 'logs',
+                                request.args['key'][0],
+                                request.args['stage'][0] + '.log')
+
+        logging.info("Opening {} as {} on {}".format(filename, user,
+                                                     section.get('server')))
+
+        try:
+            viewer = yield ssh.get_logviewer(
+                                 section.get('server'),
+                                 section.getlist('fingerprints', default=[]),
+                                 user,
+                                 self._plugin.get_private_keys(user))
+
+            file = yield viewer.openFile(str(filename),
+                                         filetransfer.FXF_READ, {})
+
+            attrs = yield defer.maybeDeferred(file.getAttrs)
+
+            request.setHeader("Content-Type", "text/plain")
+            read_bytes = 0
+            while read_bytes < attrs['size']:
+                data = yield defer.maybeDeferred(file.readChunk,
+                                                 read_bytes, _CHUNK_SIZE)
+                logging.info("Read {} bytes".format(len(data)))
+                read_bytes += len(data)
+                request.write(data)
+
+            request.finish()
+            yield defer.maybeDeferred(file.close)
+        except Exception as e:
+            logging.error("Failed to read the whole file")
+            logging.exception(e)
+            self._render_fail(request, failure.Failure(e))
+        finally:
+            viewer.transport.loseConnection()
+
+
+# Web root
+
+class KermitRoot(resource.Resource):
+    def __init__(self, plugin):
+        resource.Resource.__init__(self)
+        self._plugin = plugin
+        self.putChild("", plugin.http.authed_resource(KermitQueue(plugin)))
+        self.putChild("update", KermitUpdater(plugin))
+        self.putChild("submit", KermitSubmitter(plugin))
+        self.putChild("list", KermitLister(plugin))
+        self.putChild("stages", KermitStager(plugin))
+        self.putChild("query",
+                      plugin.http.authed_resource(KermitQuerier(plugin)))
+        self.putChild("abort", KermitAborter(plugin))
+
+        # Key management
+        self.putChild("profile",
+                      plugin.http.authed_resource(KermitProfile(plugin)))
+        self.putChild("uploadkey",
+                      plugin.http.authed_resource(KermitUploadKey(plugin)))
+        self.putChild("generatekey",
+                      plugin.http.authed_resource(KermitGenKey(plugin)))
+        self.putChild("removekey",
+                      plugin.http.authed_resource(KermitDelKey(plugin)))
+
+        # Logfile access
+        self.putChild("logfile",
+                      plugin.http.authed_resource(LogViewer(plugin)))
+
+    def render_GET(self, request):
+        return util.redirectTo(request.path + '/', request)