Merge lp:~lifeless/python-oops-tools/amqp into lp:python-oops-tools

[Robert Collins]

Add AMQP support and improve docs.

Build on the recent improvements in oops-datedir-repo and oops-amqp to add an amqp queue worker that will take oopses from amqp and load them straight into OOPS-tools. Also fix docs (0.6 is the current release) and stop telling folk to edit product.cfg in-place.

[Steve Kowalik (stevenk)]

This looks fine to me.

review: Approve (code)

[lp:~lifeless/python-oops-tools/amqp]

2. By Robert Collins on 2011-10-17

Add AMQP support and improve docs.

=== modified file 'buildout.cfg'
--- buildout.cfg	2011-10-13 20:18:51 +0000
+++ buildout.cfg	2011-10-16 23:09:17 +0000
@@ -54,7 +54,7 @@
 recipe = djangorecipe
 version = 1.3
 project = oopstools
-projectegg = oops-tools
+projectegg = oopstools
 settings = settings
 test = oopstools
 eggs = oops-tools
 
=== modified file 'setup.py'
--- setup.py	2011-10-13 20:18:51 +0000
+++ setup.py	2011-10-16 23:09:17 +0000
@@ -57,6 +57,7 @@
         'launchpadlib',
         'lazr.config',
         'oops',
+        'oops-amqp',
         'oops-datedir-repo',
         'pytz',
         'setuptools',
@@ -79,6 +80,7 @@
     ),
     entry_points=dict(
         console_scripts=[ # `console_scripts` is a magic name to setuptools
+            'amqp2disk = oopstools.scripts.amqp2disk:main',
             'analyse_error_reports = oopstools.scripts.analyse_error_reports:main',
             'load_sample_data = oopstools.scripts.load_sample_data:main',
             'update_db = oopstools.scripts.update_db:main',
 
=== modified file 'src/oopstools/NEWS.txt'
--- src/oopstools/NEWS.txt	2011-10-13 20:18:51 +0000
+++ src/oopstools/NEWS.txt	2011-10-16 23:09:17 +0000
@@ -2,8 +2,13 @@
 NEWS for oopstools
 ===================
 
-0.5 (UNRELEASED)
-================
+NEXT
+====
+
+* Added AMQP support via the bin/amqp2disk script. (Robert Collins)
+
+0.6
+===
 
 * Initial release
 
 
=== modified file 'src/oopstools/README.txt'
--- src/oopstools/README.txt	2011-10-13 20:18:51 +0000
+++ src/oopstools/README.txt	2011-10-16 23:09:17 +0000
@@ -51,19 +51,21 @@
 Deployment using mod_wsgi
 =========================
 
-Update the production.cfg file with your database configuration and the paths
-to your OOPS directories:
+Create a custom cfg file - start with production.cfg and take a copy. Update
+your copy with your database configuration and the paths to your OOPS
+directories:
 
 [configuration]
 db-name = /path/to/your/oops.db
 index-template = 'index.html'
-oopsdir = /path/to/oops/reports
+oopsdir = /path/where/rsynced/oopses/are/found
+    /another/such/path
 
 Update settings.py setting a custom SECRET_KEY
 
 To deploy oops tools make sure all the dependecies are installed.
 
- * bin/buildout -c production.cfg
+ * bin/buildout -c yourfilename.cfg
 
  * Run bin/django syncdb
 
@@ -71,6 +73,15 @@
 
  * Copy apache/oops-tools.dev.mod_wsgi to /etc/apache2/sites-available/
 
+AMQP Integration
+================
+
+The script bin/amqp2disk is an AMQP handler that will receive OOPS reports over
+AMQP and publish them locally to disk as well as loading the metadata directly
+into the oops-tools database. To use this you will need to config your OOPS
+creation to publish over AMQP. If you are using Python then the oops-amqp
+module will help you do this.
+
 Running locally
 ===============
 
 
=== modified file 'src/oopstools/oops/models.py'
--- src/oopstools/oops/models.py	2011-10-13 20:18:51 +0000
+++ src/oopstools/oops/models.py	2011-10-16 23:09:17 +0000
@@ -21,6 +21,8 @@
 import datetime
 import os.path
 
+from pytz import utc
+
 from django.db import models
 from django.db.models.signals import pre_save
 import oops_datedir_repo.serializer
@@ -278,7 +280,11 @@
     prefix = oops.get('reporter')
     if not prefix:
         # Legacy support for pre-reporter using OOPSes.
-        prefix = oops_re.match(oopsid).group('oopsprefix')
+        prefix_match = oops_re.match(oopsid)
+        if prefix_match is not None:
+            prefix = prefix_match.group('oopsprefix')
+        else:
+            prefix = 'UNKNOWN'
     prefix = prefix.upper()
     try:
         prefix = Prefix.objects.get(value__exact=prefix)
@@ -329,9 +335,10 @@
     if total_time < 0:
         total_time = 0
     # Get the oops infestation
-    exception_type = oops.get('type')
+    exception_type = oops.get('type') or ''
+    exception_value = oops.get('value') or ''
     exception_value = _normalize_exception_value(
-        exception_type, oops.get('value'), prefix)
+        exception_type, exception_value, prefix)
     try:
         infestation = Infestation.objects.get(
             exception_type__exact=exception_type,
@@ -361,10 +368,13 @@
         most_expensive_statement = conform(most_expensive_statement, 200)
     url = conform(oops.get('url') or '', MAX_URL_LEN)
     informational = oops.get('informational', 'False').lower() == 'true'
+    oops_date = oops.get('time')
+    if oops_date is None:
+        oops_date = datetime.datetime.now(utc)
     data.update(
         oopsid = oopsid,
         prefix = prefix,
-        date = oops.get('time').replace(microsecond=0),
+        date = oops_date.replace(microsecond=0),
         # Missing pageids are urls because that suits our queries.
         pageid = oops.get('topic') or url,
         url = url,
@@ -377,7 +387,28 @@
         is_bot = _robot_pat.search(data['user_agent']) is not None,
         statements_count = len(statements),
         )
-    return data, req_vars, statements, oops.get('tb_text')
+    return data, req_vars, statements, oops.get('tb_text') or ''
+
+
+def parsed_oops_to_model_oops(parsed_oops, pathname):
+    """Convert an oops report dict to an Oops object."""
+    data, req_vars, statements, traceback = _get_oops_tuple(parsed_oops)
+    data['pathname'] = pathname
+    res = Oops(**data)
+    res.appinstance = res.get_appinstance()
+    res.save()
+    # Get it again.  Otherwise we have discrepancies between old and
+    # new oops objects: old ones have unicode attributes, and new
+    # ones have string attributes, for instance.  Ideally the message
+    # conversion would have converted everything to unicode, but it
+    # doesn't easily.
+    res = Oops.objects.get(oopsid__exact=parsed_oops['id'])
+    res.parsed_oops = parsed_oops
+    res.req_vars = req_vars
+    res.statements = statements
+    res.traceback = traceback
+    res.save()
+    return res
 
 
 class Oops(models.Model):
@@ -432,21 +463,7 @@
         try:
             res = cls.objects.get(oopsid__exact=oopsid)
         except cls.DoesNotExist:
-            data, req_vars, statements, traceback = _get_oops_tuple(parsed_oops)
-            data['pathname'] = pathname
-            res = cls(**data)
-            res.appinstance = res.get_appinstance()
-            res.save()
-            # Get it again.  Otherwise we have discrepancies between old and
-            # new oops objects: old ones have unicode attributes, and new
-            # ones have string attributes, for instance.  Ideally the message
-            # conversion would have converted everything to unicode, but it
-            # doesn't easily.
-            res = cls.objects.get(oopsid__exact=oopsid)
-            res.parsed_oops = parsed_oops
-            res.req_vars = req_vars
-            res.statements = statements
-            res.traceback = traceback
+            res = parsed_oops_to_model_oops(parsed_oops, pathname)
         return res
 
     @readproperty
 
=== added file 'src/oopstools/oops/test/test_amqp2disk.py'
--- src/oopstools/oops/test/test_amqp2disk.py	1970-01-01 00:00:00 +0000
+++ src/oopstools/oops/test/test_amqp2disk.py	2011-10-16 23:09:17 +0000
@@ -0,0 +1,40 @@
+# Copyright 2005-2011 Canonical Ltd.  All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os.path
+
+import bson
+from fixtures import TempDir
+from testtools import TestCase
+
+from oopstools.oops.models import Oops
+from oopstools.scripts import amqp2disk
+
+
+class TestOOPSConfig(TestCase):
+
+    def test_publishes_disk_and_DB(self):
+        self.root_dir = self.useFixture(TempDir()).path
+        config = amqp2disk.make_amqp_config(self.root_dir)
+        orig_report = {'id': '12345'}
+        report = dict(orig_report)
+        ids = config.publish(report)
+        self.assertEqual(['12345', '12345'], ids)
+        with open(report['datedir_repo_filepath'], 'rb') as fp:
+            disk_report = bson.loads(fp.read())
+        self.assertEqual(disk_report, orig_report)
+        model_report = Oops.objects.get(oopsid='12345')
+        self.assertNotEqual(None, model_report)
 
=== added file 'src/oopstools/scripts/amqp2disk.py'
--- src/oopstools/scripts/amqp2disk.py	1970-01-01 00:00:00 +0000
+++ src/oopstools/scripts/amqp2disk.py	2011-10-16 23:09:17 +0000
@@ -0,0 +1,137 @@
+# Copyright 2011 Canonical Ltd.  All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Receive OOPS reports from AMQP and publish them into the oops-tools
+# repository.
+
+__metaclass__ = type
+
+import sys
+import optparse
+import StringIO
+from textwrap import dedent
+
+import amqplib.client_0_8 as amqp
+from oops import Config
+import oops_amqp
+import oops_datedir_repo
+
+from oopstools.oops.helpers import parsedate, load_prefixes
+from oopstools.oops.models import (
+    Oops,
+    parsed_oops_to_model_oops,
+    Prefix,
+    Report,
+    )
+from oopstools.oops.oopsstore import OopsStore
+from oopstools.oops import dbsummaries
+from oopstools.oops.summaries import (
+    WebAppErrorSummary,
+    CheckwatchesErrorSummary,
+    CodeHostingWithRemoteSectionSummary,
+    GenericErrorSummary,
+)
+
+
+def main(argv=None):
+    if argv is None:
+        argv=sys.argv
+    usage = dedent("""\
+        %prog [options]
+
+        The following options must be supplied:
+         --output
+         --host
+         --username
+         --password
+         --vhost
+         --queue
+
+        e.g.
+        amqp2disk --output /srv/oops-tools/amqpoopses --host "localhost:3472" \\
+            --username "guest" --password "guest" --vhost "/" --queue "oops"
+
+        The AMQP environment should be setup in advance with a persistent queue
+        bound to your exchange : using transient queues would allow OOPSes to
+        be lost if the amqp2disk process were to be shutdown for a non-trivial
+        duration. The --bind-to option will cause the queue to be created and
+        bound to the given exchange. This is only needed the first time as it
+        is created persistently.
+        """)
+    description = "Load OOPS reports into oops-tools from AMQP."
+    parser = optparse.OptionParser(
+        description=description, usage=usage)
+    parser.add_option('--output', help="Root directory to store OOPSes in.")
+    parser.add_option('--host', help="AQMP host / host:port.")
+    parser.add_option('--username', help="AQMP username.")
+    parser.add_option('--password', help="AQMP password.")
+    parser.add_option('--vhost', help="AMQP vhost.")
+    parser.add_option('--queue', help="AMQP queue name.")
+    parser.add_option(
+        '--bind-to', help="AMQP exchange to bind to (only needed once).")
+    options, args = parser.parse_args(argv[1:])
+    def needed(optname):
+        if getattr(options, optname, None) is None:
+            raise ValueError('option "%s" must be supplied' % optname)
+    needed('host')
+    needed('output')
+    needed('username')
+    needed('password')
+    needed('vhost')
+    needed('queue')
+    connection = amqp.Connection(host=options.host, userid=options.username,
+        password=options.password, virtual_host=options.vhost)
+    channel = connection.channel()
+    if options.bind_to:
+        channel.queue_declare(options.queue, durable=True, auto_delete=False)
+        channel.queue_bind(options.queue, options.bind_to)
+    config = make_amqp_config(options.output)
+    receiver = oops_amqp.Receiver(config, channel, options.queue)
+    try:
+        receiver.run_forever()
+    except KeyboardInterrupt:
+        pass
+
+
+def db_publisher(report):
+    """Publish OOPS reports to the oops-tools django store."""
+    # the first publisher will either inherit or assign, so this should be
+    # impossible.
+    assert report['id'] is not None
+    # Some fallback methods could lead to duplicate paths into the DB: exit
+    # early if the OOPS is already loaded.
+    try:
+        res = Oops.objects.get(oopsid__exact=report['id'])
+    except Oops.DoesNotExist:
+        res = parsed_oops_to_model_oops(
+            report, report['datedir_repo_filepath'])
+        return res.oopsid
+    return None
+
+
+def make_amqp_config(output_dir):
+    """Create an OOPS Config for republishing amqp OOPSes.
+
+    An OOPS published to this config will be written to disk and then loaded
+    into the database.
+
+    :param output_dir: The directory to write OOPSes too.
+    """
+    config = Config()
+    disk_publisher = oops_datedir_repo.DateDirRepo(
+        output_dir, inherit_id=True, stash_path=True)
+    config.publishers.append(disk_publisher.publish)
+    config.publishers.append(db_publisher)
+    return config
 
=== modified file 'versions.cfg'
--- versions.cfg	2011-10-13 20:18:51 +0000
+++ versions.cfg	2011-10-16 23:09:17 +0000
@@ -19,8 +19,9 @@
 launchpadlib = 1.6.0
 lazr.config = 1.1.3
 mechanize = 0.1.11
-oops = 0.0.7
-oops-datedir-repo = 0.0.7
+oops = 0.0.9
+oops-amqp = 0.0.1
+oops-datedir-repo = 0.0.9
 setuptools = 0.6c11
 z3c.recipe.filetemplate = 2.0.3
 z3c.recipe.sphinxdoc = 0.0.8