Merge lp:~slimey/engage/stockwell into lp:engage

[Simon C]

Embedded Twisted 12 and further ATV support

 - embedded twisted 12 for deployment on ubuntu
 - further ATV support (video support still half-baked but harmless)

[Simon C (slimey)]

review: Approve

[lp:~slimey/engage/stockwell]

4. By Simon C on 2012-09-25

Embedded Twisted 12 and further ATV support

=== added file '.bzrignore'
--- .bzrignore	1970-01-01 00:00:00 +0000
+++ .bzrignore	2012-09-25 15:49:24 +0000
@@ -0,0 +1,2 @@
+dropin.cache
+twisted-engage.egg
 
=== added directory 'bin'
=== added file 'bin/engage'
--- bin/engage	1970-01-01 00:00:00 +0000
+++ bin/engage	2012-09-25 15:49:24 +0000
@@ -0,0 +1,28 @@
+#!/usr/bin/python
+#
+# This is just a crazy equivalent of
+#
+#  /usr/bin/twistd --uid nobody --gid nogroup --logfile /var/log/engage.log --nodaemon engage
+#
+# but allowing Twisted to be embedded, for those ubutnu machines with
+# a stock (pre-12.0) version.
+#
+# Happy day when we can ditch this.
+#
+import os, platform, sys
+try:
+    import _preamble
+except ImportError:
+    sys.exc_clear()
+
+lib_dir = '/usr/lib/engage/lib-depends'
+sys.path.insert(0, lib_dir)
+sys.path.insert(0, os.path.join(lib_dir, 'twisted-engage.egg'))
+
+argv_head = sys.argv[0]
+argv_tail = sys.argv[1:]
+sys.argv = [argv_head, '--uid', 'nobody', '--gid', 'nogroup', '--logfile', 
+            '/var/log/engage.log', '--nodaemon'] + argv_tail + ['engage']
+
+from twisted.scripts.twistd import run
+run()
 
=== modified file 'debian/changelog'
--- debian/changelog	2012-09-17 13:13:06 +0000
+++ debian/changelog	2012-09-25 15:49:24 +0000
@@ -1,3 +1,9 @@
+engage (0.6.1) precise; urgency=low
+
+  * Embedded twisted 12 for easy of installing on stock ubuntu
+
+ -- Simon C <[e-mail]>  Mon, 24 Sep 2012 22:42:58 +0100
+
 engage (0.6) lucid; urgency=low
 
   * Improved structure and added basic ATV support
 
=== modified file 'debian/control'
--- debian/control	2012-09-17 13:13:06 +0000
+++ debian/control	2012-09-25 15:49:24 +0000
@@ -8,8 +8,7 @@
 Package: engage
 Architecture: all
 # snmp package is just for snmptranslate
-Depends: python,python-twisted,python-twisted-web (>= 12.0.0),python-rrdtool,libsnmp-python,python-pynetsnmp,snmp,snmp-mibs-downloader,${misc:Depends}
+Depends: python (>= ${python:Versions}), python-setuptools, python-zope.interface, python-rrdtool, libsnmp-python, python-pynetsnmp, snmp, snmp-mibs-downloader, ${python:Depends}, ${misc:Depends}
 Description: Combined system/network statistics and Apple TV control framework
  EnGage is a plugin-based framework for gathering, summarizing, and displaying 
- system or network administration data. This base package provides basic
- monitoring of the linux host itself.
+ system or network administration data.
 
=== modified file 'debian/engage.install'
--- debian/engage.install	2012-09-17 13:13:06 +0000
+++ debian/engage.install	2012-09-25 15:49:24 +0000
@@ -1,3 +1,6 @@
 etc/engage.conf etc
 etc/init/engage.conf etc/init
 static/ usr/share/engage
+bin/engage /usr/sbin
+lib-depends/twisted-engage-egg-src.tar.bz2 /usr/lib/engage/lib-depends
+lib-depends/twisted-engage.egg /usr/lib/engage/lib-depends
 
=== added file 'debian/postinst'
--- debian/postinst	1970-01-01 00:00:00 +0000
+++ debian/postinst	2012-09-25 15:49:24 +0000
@@ -0,0 +1,25 @@
+#!/bin/sh
+#
+# Build the embedded Twisted 12.2 package into an egg for the target system
+# architecture. When it's reasonable to just put Twisted >= 12.0 as a
+# dependency and expect ubuntu to honour it, we can cheerfully flush this down
+# the can.
+#
+# We include a zero-byte twisted-engage.egg and mark it as a conffile, so it's
+# ok to overwrite it here with a real egg.
+#
+set -e
+egg_src=/usr/lib/engage/lib-depends/twisted-engage-egg-src.tar.bz2
+egg=/usr/lib/engage/lib-depends/twisted-engage.egg
+tmp=/tmp/engage-$$
+rm -rf $tmp
+mkdir $tmp
+cd $tmp
+tar jxf $egg_src
+cd Twisted
+python setup.py bdist_egg
+mv dist/*.egg $egg
+cd /tmp
+rm -rf $tmp
+
+#DEBHELPER#
 
=== modified file 'etc/init/engage.conf'
--- etc/init/engage.conf	2012-09-18 21:21:43 +0000
+++ etc/init/engage.conf	2012-09-25 15:49:24 +0000
@@ -5,5 +5,5 @@
 stop on stopping network
 stop on starting shutdown
 
-exec /usr/bin/twistd --uid nobody --gid nogroup --logfile /var/log/engage.log --nodaemon engage
+exec /usr/sbin/engage
 respawn
 
=== added directory 'lib-depends'
=== added file 'lib-depends/get-egg-src'
--- lib-depends/get-egg-src	1970-01-01 00:00:00 +0000
+++ lib-depends/get-egg-src	2012-09-25 15:49:24 +0000
@@ -0,0 +1,40 @@
+#!/bin/sh
+#
+# Download twisted from pypi, add our own plugin, and then tar up so
+# it can be built on each target build.
+#
+# When we can just impose twisted >= 12.0 as a dependency and expect a
+# normal ubuntu to be able to honour that, we can trash this whole
+# sorry effort. That includes:
+#
+# - Killing this file, the twisted egg source file, and the zero-byte
+#   twisted-engage.egg file in this directory that is present just for
+#   dpkg to keep track of.
+#
+# - Killing the engage binary and just running "twistd engage" from
+#   etc/init with appropriate options.
+#
+# - Remove python-setuptools from the dependencies in debian/control
+#   and add python-twisted-core and python-twisted-web.
+#
+# - Moving engage/util/twisted_plugin.py back to twisted/plugins.
+#
+# - Adding twisted.plugins back to setup.py.
+#
+# Simon C, September 2012
+
+set -e
+src_target=$PWD/twisted-engage-egg-src.tar.bz2
+tmp=/tmp/engage-$$
+plugin_file=twisted/plugins/engage_plugin.py
+plugin_src=$PWD/../src/$plugin_file
+rm -rf $tmp
+mkdir $tmp
+cd $tmp
+pip install --upgrade --no-install Twisted
+cd build/Twisted
+cp $plugin_src $plugin_file
+(cd ..; tar cf - Twisted | bzip2 -9 > $src_target)
+cd /
+rm -rf $tmp
+ls -l $src_target
 
=== added file 'lib-depends/twisted-engage-egg-src.tar.bz2'
 Binary files lib-depends/twisted-engage-egg-src.tar.bz2	1970-01-01 00:00:00 +0000 and lib-depends/twisted-engage-egg-src.tar.bz2	2012-09-25 15:49:24 +0000 differ
=== removed file 'restart'
--- restart	2012-09-14 12:24:16 +0000
+++ restart	1970-01-01 00:00:00 +0000
@@ -1,2 +0,0 @@
-./stop $*
-./start $*
 
=== modified file 'setup.py'
--- setup.py	2012-09-18 21:21:43 +0000
+++ setup.py	2012-09-25 15:49:24 +0000
@@ -7,10 +7,10 @@
 from distutils.core import setup
 
 setup(name='engage',
-      version="0.6",
+      version="0.6.1",
       description='EnGage: system/network statistics monitoring and Apple TV control',
       url='http://open.ensoft.co.uk/EnGage',
-      packages=['engage', 'engage.collector', 'engage.monitor', 'engage.util', 
-                'engage.ui', 'twisted.plugins'],
+      packages=['engage', 'engage.collector', 'engage.media', 'engage.monitor', 
+                'engage.util', 'engage.ui'],
       package_dir={'': 'src' },
      )
 
=== added directory 'src/engage/media'
=== added file 'src/engage/media/__init__.py'
--- src/engage/media/__init__.py	1970-01-01 00:00:00 +0000
+++ src/engage/media/__init__.py	2012-09-25 15:49:24 +0000
@@ -0,0 +1,91 @@
+# Media base class
+#
+# Simon C, September 2012
+
+#
+# Unofficial AirPlay specification:
+#
+# http://nto.github.com/AirPlay.html
+#
+
+from zope.interface import Interface, implements
+from twisted.internet import reactor
+from twisted.plugin import IPlugin, getPlugins
+from twisted.application import service
+from twisted.python import log
+import engage.media
+
+class MediaSource(object):
+    """
+    Base classes for a source of media to be displayed on a carousel
+    """
+
+    cfg_defaults = {}
+
+    def __init__(self, name, cfg, atv, register_cb):
+        """
+        Initialize with the configuration for this media source. Call
+        back register_cb with the number of items this media source
+        can render (and call again whenever this updates).
+        """
+        self.name = '%s (%s)' % (name, self.__class__.__name__)
+        self.cfg = cfg.copy()
+        for k, v in self.cfg_defaults.items():
+            if k not in self.cfg:
+                self.cfg[k] = v
+        log.msg("Starting media source %s with config %s" % (self.name, self.cfg))
+        self.atv = atv
+        self.register_cb = register_cb
+        reactor.callLater(0, self.register_items)
+
+    def register_items(self):
+        """
+        For documentation only in the base class. Work out how many
+        different items we could render, and tell the carousel that.
+        """
+
+    def show(self):
+        """
+        For documentation only in the base class. Show a random media
+        object using the provided http_agent. Return a deferred that
+        fires when done.
+        """
+
+    @classmethod
+    def config_start(cls, cfg, ip_addr, register_cb):
+        """
+        Start all the media sources that are configured
+        """
+        media_sources = {}
+        for media_source in getPlugins(IMediaSource, engage.media):
+            for source in media_source.config_start(cfg, ip_addr, register_cb):
+                media_sources[source] = source.name
+        return media_sources
+
+class IMediaSource(Interface):
+    """
+    Twisted plugin specification for a config-started MediaSource
+    """
+    def config_start(self, cfg, atv, register_cb):
+        """
+        Create a MedaSource if configured
+        """
+
+class StartMediaSource(object):
+    """
+    Twisted plugin for starting a MediaSource
+    """
+    implements(IPlugin, IMediaSource)
+
+    def __init__(self, stanza_name, media_source_cls):
+        self.stanza_name = stanza_name
+        self.media_source_cls = media_source_cls
+
+    def config_start(self, cfg, atv, register_cb):
+        started = []
+        for key in cfg:
+            if key.startswith(self.stanza_name):
+                log.msg('Starting media source %s from config stanza %s' %
+                        (self.media_source_cls.__name__, key))
+                started.append(self.media_source_cls(key, cfg[key], atv, register_cb))
+        return started
 
=== added file 'src/engage/media/movie.py'
--- src/engage/media/movie.py	1970-01-01 00:00:00 +0000
+++ src/engage/media/movie.py	2012-09-25 15:49:24 +0000
@@ -0,0 +1,21 @@
+# Static file-drive media source
+#
+# Simon C, September 2012
+
+from twisted.python import log
+from engage.media import MediaSource, StartMediaSource
+
+#
+# @@@ still working out structure here, this is probably all wrong
+#
+
+class MovieMediaSource(MediaSource):
+
+    def register_items(self):
+        self.register_cb(self, 1)
+
+    def show(self):
+        d = self.atv.show_movie('http://172.22.22.143:8421/mp4/amy-janet.mp4')
+        return d
+
+start = StartMediaSource('movie', MovieMediaSource)
 
=== added file 'src/engage/media/static.py'
--- src/engage/media/static.py	1970-01-01 00:00:00 +0000
+++ src/engage/media/static.py	2012-09-25 15:49:24 +0000
@@ -0,0 +1,36 @@
+# Static file-drive media source
+#
+# Simon C, September 2012
+
+import os, random, re
+from twisted.internet import reactor
+from twisted.python import log
+from engage.media import MediaSource, StartMediaSource
+
+class StaticMediaSource(MediaSource):
+
+    cfg_defaults = {
+        'refresh': '20',
+        'weight': '1.0',
+        }
+
+    def register_items(self):
+        self.photos = []
+        for folder, _, files in os.walk(self.cfg['photo_dir'], followlinks=True):
+            for filename in files:
+                if re.search('(?i)\.(jpg|png)', filename):
+                    pathname = os.path.join(folder, filename)
+                    self.photos.append(pathname)
+        log.msg("%s found %d photos to display with weight %s" % 
+                (self.name, len(self.photos), self.cfg['weight']))
+        self.register_cb(self, len(self.photos) * float(self.cfg['weight']))
+        # re-scan every 20 minutes by default
+        reactor.callLater(60 * float(self.cfg['refresh']), self.register_items)
+
+    def show(self):
+        item = random.randint(0, len(self.photos) - 1)
+        pathname = self.photos[item]
+        d = self.atv.show_photo(pathname)
+        return d
+
+start = StartMediaSource('static', StaticMediaSource)
 
=== modified file 'src/engage/monitor/percent_used.py'
--- src/engage/monitor/percent_used.py	2012-09-18 21:21:43 +0000
+++ src/engage/monitor/percent_used.py	2012-09-25 15:49:24 +0000
@@ -32,9 +32,9 @@
 
         # Memory
         with open('/proc/meminfo') as f:
-            total, free, buffers = [int(f.readline().split()[1]) 
-                                    for i in xrange(0, 3)]
-            mem_used = percent(total - free - buffers, total)
+            total, free, buffers, cached = \
+	    	[int(f.readline().split()[1]) for i in xrange(0, 4)]
+            mem_used = percent(total - free - buffers - cached, total)
             mem_buff = percent(buffers, total)
 
         # Disk
 
=== modified file 'src/engage/ui/carousel.py'
--- src/engage/ui/carousel.py	2012-09-18 21:21:43 +0000
+++ src/engage/ui/carousel.py	2012-09-25 15:49:24 +0000
@@ -6,9 +6,9 @@
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
-from twisted.web.client import Agent, FileBodyProducer, HTTPConnectionPool
-from twisted.web.http_headers import Headers
 from engage.monitor.traffic import TrafficMonitor
+from engage.media import MediaSource
+from engage.util.atv import ATV
 
 class Carousel(object):
     """
@@ -36,13 +36,11 @@
         self.cfg = carousel_cfg
         self.period = int(carousel_cfg.get('period', 7))
         self.threshold = int(carousel_cfg.get('threshold', 10000))
-        self.url = 'http://%s:7000/photo' % carousel_cfg['atv']
-        self.agent = Agent(reactor, pool=HTTPConnectionPool(reactor))
-        self.headers = Headers({
-                'User-Agent': ['MediaControl/1.0'],
-                'X-Apple-Transition': ['Dissolve']})
-        self.scan_photo_dir()
-        self.photo_idx = 0
+        self.atv = ATV(carousel_cfg['atv'])
+        self.total_weight = 0
+        self.media_weights = {}
+        self.media_source_names = MediaSource.config_start(
+            carousel_cfg, self.atv, self.register_media_source)
 
         # Get initial SNMP stats
         self.snmp_results = (0, 0, 0) # fallback for failure
@@ -54,6 +52,14 @@
         # Kick off periodic photo display
         reactor.callLater(self.period, self.maybe_show_next_photo)
 
+    def register_media_source(self, media_source, num_items):
+        self.media_weights[media_source] = num_items
+        self.total_weight = 0
+        for weight in self.media_weights.values():
+            self.total_weight += weight
+        log.msg("Registered %d out of %d items from %s" %
+                (num_items, self.total_weight, self.media_source_names[media_source]))
+
     def maybe_show_next_photo(self):
         #
         # Main workhorse of this class, that is called periodically.
@@ -76,8 +82,10 @@
             log.msg("Carousel failure: %s" % fail)
             failure.printTraceback()
         d.addErrback(handle_failure)
-        d.addBoth(lambda r: 
-                  reactor.callLater(self.period, self.maybe_show_next_photo))
+        def reset(result):
+            log.msg("FINAL CALLBACK!!!")
+            reactor.callLater(self.period, self.maybe_show_next_photo)
+        d.addBoth(reset)
         
     def check_snmp_results(self, results):
         #
@@ -93,18 +101,28 @@
 
     def conditionally_show_next_photo(self, tx_bps):
         #
-        # If the tx bps is low enough, pick a photo and send it.
-        # Currently very dumb: round robin
+        # If the tx bps is low enough, pick a media source at random
+        # and ask it to do something.
         #
         if tx_bps >= self.threshold:
             log.msg("Skipping ATV photo, tx bps = %d" % tx_bps)
+            self.atv.drop_connection()
         else:
-            pathname = self.photos[self.photo_idx]
-            log.msg("Showing photo %s (tx bps = %d)" % (pathname, tx_bps))
-            body = FileBodyProducer(file(pathname))
-            d = self.agent.request('PUT', self.url, self.headers, body)
-            d.addCallbacks(self.display_done, self.display_failed)
-            return d
+            r = random.randint(0, self.total_weight)
+            media_source = None
+            for possible_source in self.media_source_names:
+                media_source = possible_source
+                r -= self.media_weights[possible_source]
+                if r < 0:
+                    break
+            if media_source is None:
+                log.msg("No media source")
+            else:
+                log.msg("Using media source %s" % (self.media_source_names[media_source]))
+                d = media_source.show()
+                d.addCallback(lambda r: "MEDIA SOURCE SHOW CALLBACK!!!")
+                d.addErrback(self.display_failed)
+                return d
 
     def get_second_snmp_results(self, _):
         #
@@ -125,35 +143,6 @@
         log.msg("Display failed: %s" % failure)
         failure.printTraceback()
             
-    def display_done(self, response):
-        # 
-        # Something went wrong sending a photo to the ATV - log it
-        #
-        if response.code != 200:
-            log.msg("Bad display response: key %s, code %s, phrase %s" % 
-                    (response.version, response.code, response.phrase))
-
-        # Either way, pick a new photo at random
-        new_photo_idx = self.photo_idx
-        while new_photo_idx == self.photo_idx:
-            new_photo_idx = random.randint(0, len(self.photos) - 1)
-        self.photo_idx = new_photo_idx
-
-    def scan_photo_dir(self):
-        #
-        # Scan the configured path for any plausible photos. Re-scan
-        # sometimes in case new photos have arrived.
-        #
-        self.photos = []
-        for folder, _, files in os.walk(self.cfg['photo_dir']):
-            for filename in files:
-                if re.search('(?i)\.jpg', filename):
-                    pathname = os.path.join(folder, filename)
-                    self.photos.append(pathname)
-        log.msg("Found %d photos to display after scan" % len(self.photos))
-        # re-scan every 20 minutes
-        reactor.callLater(60 * 20, self.scan_photo_dir)
-
 class CarouselService(service.Service):
     """
     Twisted Service encapsulating one carousel
 
=== modified file 'src/engage/ui/web.py'
--- src/engage/ui/web.py	2012-09-18 21:21:43 +0000
+++ src/engage/ui/web.py	2012-09-25 15:49:24 +0000
@@ -159,6 +159,8 @@
     site.putChild("", Main(config))
     site.putChild("graph", Graph(config))
     site.putChild("png", File('%s/png' % web_config['web_static_dir']))
+    site.putChild("mp4", File('%s/mp4' % web_config['web_static_dir']))
+
 
     web = Site(site)
     tcp_service = internet.TCPServer(int(web_config['http_port']), web)
 
=== added file 'src/engage/util/atv.py'
--- src/engage/util/atv.py	1970-01-01 00:00:00 +0000
+++ src/engage/util/atv.py	2012-09-25 15:49:24 +0000
@@ -0,0 +1,94 @@
+# Apple TV control
+#
+# Simon C, September 2012
+
+import re
+from StringIO import StringIO
+from twisted.internet import defer, reactor
+from twisted.python import log
+from twisted.internet.protocol import Protocol
+from twisted.web.client import Agent, HTTPConnectionPool
+from twisted.web.client import FileBodyProducer
+from twisted.web.http_headers import Headers
+
+POLL_INTERVAL = 1
+
+class ResponseReceiver(Protocol):
+
+    def __init__(self, finished, fire_again):
+        self.finished = finished
+        self.fire_again = fire_again
+        self.remaining = POLL_INTERVAL
+
+    def dataReceived(self, data):
+        position, duration = [float(re.search(key + ': ([\d.]+)', data).group(1))
+                              for key in ('position', 'duration')]
+        if duration > 0:
+            remaining = duration - position
+        if remaining < self.remaining:
+            self.remaining = remaining
+
+        log.msg("Video at %f/%f; timer reset to %f" %
+                (position, duration, self.remaining))
+
+    def connectionLost(self, reason):
+        if self.remaining < POLL_INTERVAL:
+            log.msg("Last %f seconds, just waiting" % self.remaining)
+        else:
+            self.finished.addCallback(self.fire_again)
+        reactor.callLater(self.remaining, self.finished.callback, None)
+
+
+class ATV(object):
+
+    HEADERS = Headers({'User-Agent': ['MediaControl/1.0'],
+                       'X-Apple-Transition': ['Dissolve']})
+
+    def __init__(self, ip_addr):
+        self.http_agent = None
+        self.base_url = 'http://%s:7000' % ip_addr
+
+    def drop_connection(self):
+        self.http_agent = None
+
+    def _check_connection(self):
+        if self.http_agent is None:
+            self.http_agent = Agent(reactor, pool=HTTPConnectionPool(reactor))
+
+    def show_photo(self, pathname):
+        self._check_connection()
+        body = FileBodyProducer(file(pathname))
+        d = self.http_agent.request('PUT', self.base_url + '/photo', 
+                                    self.HEADERS, body)
+        def display_done(response):
+            if response.code != 200:
+                log.msg("Bad response from photo %s: key %s, code %s, phrase %s" % 
+                        (pathname, response.version, response.code, response.phrase))
+        d.addCallback(display_done)
+        return d
+
+    def show_movie(self, url):
+        self._check_connection()
+        body_string = 'Content-Location: %s\nStart-Position: 0\n' % url
+        body = FileBodyProducer(StringIO(body_string))
+        url = self.base_url + '/play'
+        d = self.http_agent.request('POST', url, self.HEADERS, body)
+        d.addCallback(self.get_scrub_soon)
+        return d
+
+    def get_scrub_soon(self, result):
+        d = defer.Deferred()
+        d.addCallback(self.get_scrub)
+        reactor.callLater(POLL_INTERVAL, lambda: d.callback(result))
+        return d
+
+    def get_scrub(self, result):
+        url = self.base_url + '/scrub'
+        d = self.http_agent.request('GET', url)
+        d.addCallback(self.display_scrub)
+        return d
+
+    def display_scrub(self, response):
+        d = defer.Deferred()
+        response.deliverBody(ResponseReceiver(d, self.get_scrub))
+        return d
 
=== added file 'src/engage/util/twisted_plugin.py'
--- src/engage/util/twisted_plugin.py	1970-01-01 00:00:00 +0000
+++ src/engage/util/twisted_plugin.py	2012-09-25 15:49:24 +0000
@@ -0,0 +1,65 @@
+# EnGage twisted plugin
+#
+# Simon C, September 2012
+
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+from twisted.application import service
+
+from engage.collector.rrd import RRDCollector
+from engage.ui import web, carousel
+from engage.util import config
+
+class Options(usage.Options):
+
+    optParameters = [
+        ('conf', 'c', '/etc/engage.conf', 'EnGage configuration file.'),
+        ]
+
+#
+# Built-in configuration. We don't really want the admin messing with the first 
+# two; the others are more just handy defaults. 
+#
+collector_cfg = {
+    'interval': 10,
+    'rrd_dir': '/var/lib/engage/rrd',
+}
+
+web_cfg = {
+    'web_static_dir': '/usr/share/engage/static',
+    'http_prefix': '',
+    'http_port': '8421',
+    'title': 'EnGage',
+}
+
+class EngageServiceMaker(object):
+
+    implements(service.IServiceMaker, IPlugin)
+
+    tapname = "engage"
+    description = "EnGage system/network monitoring service."
+    options = Options
+
+    def makeService(self, options):
+        top_service = service.MultiService()
+
+        # Merge the user and built-in configuration
+        all_config = config.parse(options['conf'])
+        for k, v in all_config['Main'].items():
+            web_cfg[k] = v
+            collector_cfg[k] = v
+
+        # Start monitors based on configuration, with data collected by RRD
+        RRDCollector.config_start_monitors(collector_cfg, 
+                                           all_config,
+                                           top_service)
+
+        # Start carousel if configured
+        if 'Carousel' in all_config:
+            carousel.start(all_config, top_service)
+        
+        # Start the web server too
+        web.start(web_cfg, top_service)
+
+        return top_service
 
=== modified file 'src/twisted/plugins/engage_plugin.py'
--- src/twisted/plugins/engage_plugin.py	2012-09-18 21:21:43 +0000
+++ src/twisted/plugins/engage_plugin.py	2012-09-25 15:49:24 +0000
@@ -1,67 +1,6 @@
-# EnGage twisted plugin
-#
-# Simon C, September 2012
-
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from twisted.application import service
-
-from engage.collector.rrd import RRDCollector
-from engage.ui import web, carousel
-from engage.util import config
-
-class Options(usage.Options):
-
-    optParameters = [
-        ('conf', 'c', '/etc/engage.conf', 'EnGage configuration file.'),
-        ]
-
-#
-# Built-in configuration. We don't really want the admin messing with the first 
-# two; the others are more just handy defaults. 
-#
-collector_cfg = {
-    'interval': 10,
-    'rrd_dir': '/var/lib/engage/rrd',
-}
-
-web_cfg = {
-    'web_static_dir': '/usr/share/engage/static',
-    'http_prefix': '',
-    'http_port': '8421',
-    'title': 'EnGage',
-}
-
-class EngageServiceMaker(object):
-
-    implements(service.IServiceMaker, IPlugin)
-
-    tapname = "engage"
-    description = "EnGage system/network monitoring service."
-    options = Options
-
-    def makeService(self, options):
-        top_service = service.MultiService()
-
-        # Merge the user and built-in configuration
-        all_config = config.parse(options['conf'])
-        for k, v in all_config['Main'].items():
-            web_cfg[k] = v
-            collector_cfg[k] = v
-
-        # Start monitors based on configuration, with data collected by RRD
-        RRDCollector.config_start_monitors(collector_cfg, 
-                                           all_config,
-                                           top_service)
-
-        # Start carousel if configured
-        if 'Carousel' in all_config:
-            carousel.start(all_config, top_service)
-        
-        # Start the web server too
-        web.start(web_cfg, top_service)
-
-        return top_service
-
+#
+# Crazy file to allow embedding of Twisted as an egg, for stock Ubuntu
+# distributions with a pre-12.0.0 version
+#
+from engage.util.twisted_plugin import EngageServiceMaker
 service_maker = EngageServiceMaker()
 
=== removed file 'start'
--- start	2012-09-14 12:24:16 +0000
+++ start	1970-01-01 00:00:00 +0000
@@ -1,19 +0,0 @@
-hostname=$(hostname)
-if [ z$ENGAGE == z ]; then
-    engage=engage
-    instance=$hostname
-else
-    engage=test_engage
-    instance=test_$hostname
-fi
-CFGFILE=$PWD/etc/$instance.conf
-PIDFILE=$PWD/var/run/$engage.pid
-LOGFILE=$PWD/var/log/$instance.log
-
-if [ ! -f $CFGFILE ]; then
-  echo No specific config file for host $hostname, using default
-  CFGFILE=$PWD/etc/engage.conf
-fi
-
-cd src
-twistd $* --pidfile $PIDFILE --logfile $LOGFILE engage -c $CFGFILE
 
=== removed file 'stop'
--- stop	2012-09-14 12:24:16 +0000
+++ stop	1970-01-01 00:00:00 +0000
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-if [ z$ENGAGE == z ]; then
-    engage=engage
-    rrd_dir=var/rrd
-else
-    engage=test_engage
-    rrd_dir=var/test-rrd
-fi
-PIDFILE=$PWD/var/run/$engage.pid
-
-if [ -f $PIDFILE ]; then
-  cd $rrd_dir
-  # pause for most recent data collection, to avoid holes in data set
-  perl -e '$| = 1; while (time - (stat((glob("*.rrd"))[0]))[9] != 1) { print "."; sleep 1; }; print "\n"'
-  kill $(cat $PIDFILE)
-fi