Merge lp:~lifeless/python-oops-tools/prune into lp:python-oops-tools

[Robert Collins]

Permit pruning of OOPS from the DB that are not referenced by a bug report after a week.

This implements oops pruning building on the pruning present in datedir-repo. The custom code for python-oops-tools, that isn't part of the CLI, is uhm trivial. I've got room to add tests, but they seemed of vanishing utility here so I'm going to throw myself on the mercy of my reviewer.

[William Grant (wgrant)]

+ oldest_oops = Oops.objects.order_by('id')[0]

id isn't used as a criterion anywhere else. Perhaps sort by date?

review: Approve (code)

[Robert Collins (lifeless)]

we want the oldest oops in the system - there has been no other cause for that particular query to date.

[lp:~lifeless/python-oops-tools/prune]

21. By Robert Collins on 2011-11-21

Print out the path to the OOPS when amqp2disk fails to import it.

22. By Robert Collins on 2011-11-21

Permit wider prefix and appinstance fields.

23. By Robert Collins on 2011-11-21

Permit pruning of OOPS from the DB that are not referenced by a bug report after a week.

=== modified file '.bzrignore'
--- .bzrignore	2011-10-13 20:18:51 +0000
+++ .bzrignore	2011-11-21 04:15:28 +0000
@@ -14,3 +14,4 @@
 apache/*
 src/oopstools/settings.py
 ./dist
+prune.log

=== modified file 'setup.py'
--- setup.py	2011-10-16 22:35:01 +0000
+++ setup.py	2011-11-21 04:15:28 +0000
@@ -83,6 +83,7 @@
             'amqp2disk = oopstools.scripts.amqp2disk:main',
             'analyse_error_reports = oopstools.scripts.analyse_error_reports:main',
             'load_sample_data = oopstools.scripts.load_sample_data:main',
+            'prune = oopstools.scripts.prune:main',
             'update_db = oopstools.scripts.update_db:main',
             'dir_finder = oopstools.scripts.dir_finder:main',
             'report = oopstools.scripts.report:main',

=== modified file 'src/oopstools/NEWS.txt'
--- src/oopstools/NEWS.txt	2011-11-16 23:21:38 +0000
+++ src/oopstools/NEWS.txt	2011-11-21 04:15:28 +0000
@@ -21,6 +21,9 @@
 * Mixed case OOPS reports can now be looked up in the web UI without their
   OOPS-prefix (if they had one). (Robert Collins, #884571)
 
+* Old OOPS reports can be cleaned out using the new script bin/prune.
+  (Robert Collins)
+
 * OOPS reports that don't meet the normal rules for req_vars are handled
   a bit better.
   (Robert Collins, William Grant, Roman Yepishev, #885416, #884265)

=== added file 'src/oopstools/oops/migrations/0020_add_oops_prune.py'
--- src/oopstools/oops/migrations/0020_add_oops_prune.py	1970-01-01 00:00:00 +0000
+++ src/oopstools/oops/migrations/0020_add_oops_prune.py	2011-11-21 04:15:28 +0000
@@ -0,0 +1,40 @@
+# Copyright 2011 Canonical Ltd.  All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from south.v2 import DataMigration
+from south.db import db
+
+
+class Migration(DataMigration):
+
+    def forwards(self, orm):
+        db.create_table('oops_pruneinfo', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('pruned_until', self.gf('django.db.models.fields.DateTimeField')())
+            ))
+        db.send_create_signal('oops', ['PruneInfo'])
+
+    def backwards(self, orm):
+        db.delete_table('oops_pruneinfo')
+
+    models = {
+        'oops.prune': {
+            'Meta': {'object_name': 'PruneInfo'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'pruned_until': ('django.db.models.fields.DateTimeField', [], {}),
+        },
+    }
+    
+    complete_apps = ['oops']

=== modified file 'src/oopstools/oops/models.py'
--- src/oopstools/oops/models.py	2011-11-17 16:11:12 +0000
+++ src/oopstools/oops/models.py	2011-11-21 04:15:28 +0000
@@ -120,6 +120,25 @@
         return unicode(self.value)
 
 
+class PruneInfo(models.Model):
+    """Information about oops pruning."""
+    
+    pruned_until = models.DateTimeField()
+
+    @classmethod
+    def prune_unreferenced(klass, prune_from, prune_until, references):
+        # XXX: This trusts date more than we may want to - should consider
+        # having a date_received separate to the date generated.
+        to_delete = set(Oops.objects.filter(
+            date__gte=prune_from, date__lte=prune_until).exclude(
+            oopsid__in=references))
+        # deleting 1 at a time is a lot of commits, but leaves the DB lively
+        # for other transactions. May need to batch this in future if its
+        # too slow.
+        for oopsid in to_delete:
+            Oops.objects.filter(oopsid__exact=oopsid).delete()
+
+
 class Report(models.Model):
     """A summary report for OOPSes of a set of prefixes."""
 

=== added file 'src/oopstools/scripts/prune.py'
--- src/oopstools/scripts/prune.py	1970-01-01 00:00:00 +0000
+++ src/oopstools/scripts/prune.py	2011-11-21 04:15:28 +0000
@@ -0,0 +1,123 @@
+# Copyright 2011 Canonical Ltd.  All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Delete DB records of OOPSes that have no bug reports for them.
+
+__metaclass__ = type
+
+import datetime
+import logging
+import optparse
+import sys
+from textwrap import dedent
+
+from oops_datedir_repo.prune import LaunchpadTracker
+from pytz import utc
+
+from oopstools.oops.models import (
+    Oops,
+    PruneInfo,
+    )
+
+
+def main(argv=None, tracker=LaunchpadTracker, logging=logging):
+    """Console script entry point."""
+    if argv is None:
+        argv = sys.argv
+    usage = dedent("""\
+        %prog [options]
+
+        The following options must be supplied:
+        Either
+         --project
+        or
+         --projectgroup
+
+        e.g.
+        %prog --projectgroup launchpad-project
+
+        Will process every member project of launchpad-project.
+
+        When run this program will ask Launchpad for OOPS references made since
+        the last date it pruned up to, with an upper limit of one week from
+        today. It then looks in the database for all oopses created during that
+        date range, and if they are not in the set returned by Launchpad,
+        deletes them. If the database has never been pruned before, it will
+        pick the earliest date present in the repository as the start date.
+
+        This does not delete OOPS files that are on disk (e.g. created by
+        amqp2disk) - a separate pruner is available in oops-datedir-repo that
+        should be used to prune them.
+        """)
+    description = \
+        "Delete OOPS reports that are not referenced in a bug tracker."
+    parser = optparse.OptionParser(
+        description=description, usage=usage)
+    parser.add_option('--project',
+        help="Launchpad project to find references in.")
+    parser.add_option('--projectgroup',
+        help="Launchpad project group to find references in.")
+    parser.add_option(
+        '--lpinstance', help="Launchpad instance to use", default="production")
+    options, args = parser.parse_args(argv[1:])
+    def needed(*optnames):
+        present = set()
+        for optname in optnames:
+            if getattr(options, optname, None) is not None:
+                present.add(optname)
+        if not present:
+            if len(optnames) == 1:
+                raise ValueError('Option "%s" must be supplied' % optname)
+            else:
+                raise ValueError(
+                    'One of options %s must be supplied' % (optnames,))
+        elif len(present) != 1:
+            raise ValueError(
+                    'Only one of options %s can be supplied' % (optnames,))
+    needed('project', 'projectgroup')
+    logging.basicConfig(
+        filename='prune.log', filemode='w', level=logging.DEBUG)
+    one_week = datetime.timedelta(weeks=1)
+    one_day = datetime.timedelta(days=1)
+    # Only prune OOPS reports more than one week old.
+    prune_until = datetime.datetime.now(utc) - one_week
+    # Ignore OOPS reports we already found references for - older than the last
+    # prune date.
+    try:
+        info = PruneInfo.objects.all()[0]
+    except IndexError:
+        # Never been pruned.
+        try:
+            oldest_oops = Oops.objects.order_by('id')[0]
+        except IndexError:
+            # And has no oopses
+            return 0
+        info = PruneInfo(pruned_until=oldest_oops.date-one_day)
+        info.save()
+    prune_from = info.pruned_until
+    if prune_from.tzinfo is None:
+        # Workaround django tz handling bug:
+        # https://code.djangoproject.com/ticket/17062
+        prune_from = prune_from.replace(tzinfo=utc)
+    # The tracker finds all the references for the selected dates.
+    finder = tracker(options)
+    references = finder.find_oops_references(
+        prune_from, prune_until, options.project, options.projectgroup)
+    # Then we can delete the unreferenced oopses.
+    PruneInfo.prune_unreferenced(prune_from, prune_until, references)
+    # And finally save the fact we have scanned up to the selected date.
+    info.pruned_until = prune_until
+    info.save()
+    return 0

=== modified file 'versions.cfg'
--- versions.cfg	2011-11-16 07:25:56 +0000
+++ versions.cfg	2011-11-21 04:15:28 +0000
@@ -21,7 +21,7 @@
 mechanize = 0.1.11
 oops = 0.0.10
 oops-amqp = 0.0.4
-oops-datedir-repo = 0.0.12
+oops-datedir-repo = 0.0.14
 setuptools = 0.6c11
 z3c.recipe.filetemplate = 2.0.3
 z3c.recipe.sphinxdoc = 0.0.8