Merge lp:~michael.nelson/ubuntu-webcatalog/788207-include-ratings-stats into lp:ubuntu-webcatalog

[Michael Nelson]

Add import_ratings_stats management command.

Overview
========
This is the first branch for bug 813346 - displaying ratings stats for each app. It defines a management command to pull in the review stats for a given distroseries and update the application objects (with new attributes defined in the migration).

The next branch will add the stats to the UI.

To test:
follow the readme to bootstrap and then `fab test`

[lp:~michael.nelson/ubuntu-webcatalog/788207-include-ratings-stats]

52. By Michael Nelson on 2011-07-20

Updated new_file_template removing unnecessary cruft (according to mfoord).

[Michael Hall (mhall119)]

There isn't actually a README file

[Michael Foord (mfoord)]

Nice tests.

review: Approve

=== modified file '.bzrignore'
--- .bzrignore	2011-07-06 13:53:41 +0000
+++ .bzrignore	2011-07-20 15:32:34 +0000
@@ -10,3 +10,4 @@
 branches/
 sourcecode/framework/
 sourcecode/django/
+django_project/rnrclient.py

=== modified file 'django_project/config/main.cfg'
--- django_project/config/main.cfg	2011-07-05 16:09:31 +0000
+++ django_project/config/main.cfg	2011-07-20 15:32:34 +0000
@@ -98,3 +98,5 @@
 [google]
 google_analytics_id = UA-1018242-24
 
+[rnr]
+rnr_service_root = http://reviews.ubuntu.com/reviews/api/1.0

=== modified file 'fabtasks/bootstrap.py'
--- fabtasks/bootstrap.py	2011-07-05 16:09:31 +0000
+++ fabtasks/bootstrap.py	2011-07-20 15:32:34 +0000
@@ -112,6 +112,12 @@
     _symlink(
         "branches/django-openid-auth/django_openid_auth",
         "django_project/django_openid_auth")
+    _get_or_pull_bzr_branch(
+        "lp:/~rnr-developers/rnr-server/rnrclient",
+        "rnrclient")
+    _symlink(
+        "branches/rnrclient/rnrclient.py",
+        "django_project/rnrclient.py")
 
 def bootstrap():
     virtualenv_create()

=== modified file 'new_file_template.py'
--- new_file_template.py	2011-04-07 08:57:13 +0000
+++ new_file_template.py	2011-07-20 15:32:34 +0000
@@ -16,12 +16,3 @@
 #   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """XXX: Module docstring goes here."""
-
-from __future__ import (
-    absolute_import,
-    with_statement,
-    )
-
-__metaclass__ = type
-__all__ = [
-    ]

=== modified file 'requirements.txt'
--- requirements.txt	2011-06-28 10:29:23 +0000
+++ requirements.txt	2011-07-20 15:32:34 +0000
@@ -8,6 +8,7 @@
 django-preflight
 mock
 PIL
+piston-mini-client
 python-debian
 python-openid
 setuptools

=== modified file 'src/webcatalog/management/commands/import_app_install_data.py'
--- src/webcatalog/management/commands/import_app_install_data.py	2011-07-07 14:15:04 +0000
+++ src/webcatalog/management/commands/import_app_install_data.py	2011-07-20 15:32:34 +0000
@@ -241,7 +241,7 @@
 
         prefetched_apps = dict((app.package_name, app) for app in
             Application.objects.filter(distroseries=distroseries))
-        
+
         # For each package in the apt-cache, we update (or possibly
         # create) a matching db entry.
         for package in cache:

=== added file 'src/webcatalog/management/commands/import_ratings_stats.py'
--- src/webcatalog/management/commands/import_ratings_stats.py	1970-01-01 00:00:00 +0000
+++ src/webcatalog/management/commands/import_ratings_stats.py	2011-07-20 15:32:34 +0000
@@ -0,0 +1,98 @@
+#   -*- coding: utf-8 -*-
+#   This file is part of the Ubuntu Web Catalog
+#   Copyright (C) 2011 Canonical Ltd.
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Management command to import review statistics for a distroseries."""
+
+import os
+import shutil
+import tempfile
+import urllib
+import operator
+from datetime import (
+    datetime,
+    timedelta,
+    )
+
+from django.conf import settings
+from django.core.management.base import LabelCommand
+
+from rnrclient import RatingsAndReviewsAPI
+
+from webcatalog.models import (
+    Application,
+    DistroSeries,
+    ReviewStatsImport,
+    )
+
+
+class Command(LabelCommand):
+
+    help = "Update application review statistics."
+    verbosity = 1
+
+    def handle_label(self, distroseries_name, **options):
+        self.verbosity = int(options['verbosity'])
+        # Check when we most recently imported review stats for this
+        # distroseries.
+        distroseries, created = DistroSeries.objects.get_or_create(
+            code_name=distroseries_name)
+
+        stats = self.download_review_stats(distroseries)
+
+        self.update_apps_with_stats(distroseries, stats)
+
+        self.update_last_import_timestamp(distroseries)
+
+    def update_apps_with_stats(self, distroseries, stats):
+        stats_dict = dict((stat.package_name, stat) for stat in stats)
+        apps = Application.objects.filter(distroseries=distroseries,
+            package_name__in=stats_dict.keys()).order_by('package_name')
+
+        for app in apps:
+            stat = stats_dict[app.package_name]
+            app.ratings_average = stat.ratings_average
+            app.ratings_total = stat.ratings_total
+            app.ratings_histogram = stat.histogram
+            app.save()
+
+    def update_last_import_timestamp(self, distroseries):
+        review_stats_import, created = ReviewStatsImport.objects.get_or_create(
+            distroseries=distroseries)
+        review_stats_import.last_import = datetime.utcnow()
+        review_stats_import.save()
+
+    def download_review_stats(self, distroseries):
+        # Download the appropriate stats based on above.
+        other_imports = ReviewStatsImport.objects.filter(
+            distroseries=distroseries).order_by('-last_import')
+        num_days = None
+        if other_imports:
+            stats_import = other_imports[0]
+            time_since_last_import = (
+                datetime.utcnow() - stats_import.last_import)
+            if time_since_last_import < timedelta(days=1):
+                num_days = 1
+            elif time_since_last_import < timedelta(days=3):
+                num_days = 3
+            elif time_since_last_import < timedelta(days=7):
+                num_days = 7
+
+        rnr_api = RatingsAndReviewsAPI(service_root=settings.RNR_SERVICE_ROOT)
+        kwargs = dict(origin='ubuntu', distroseries=distroseries.code_name)
+        if num_days:
+            kwargs['days'] = num_days
+        return rnr_api.review_stats(**kwargs)

=== added file 'src/webcatalog/migrations/0006_add_review_stats.py'
--- src/webcatalog/migrations/0006_add_review_stats.py	1970-01-01 00:00:00 +0000
+++ src/webcatalog/migrations/0006_add_review_stats.py	2011-07-20 15:32:34 +0000
@@ -0,0 +1,92 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'ReviewStatsImport'
+        db.create_table('webcatalog_reviewstatsimport', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('distroseries', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['webcatalog.DistroSeries'])),
+            ('last_import', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.utcnow)),
+        ))
+        db.send_create_signal('webcatalog', ['ReviewStatsImport'])
+
+        # Adding field 'Application.ratings_total'
+        db.add_column('webcatalog_application', 'ratings_total', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True), keep_default=False)
+
+        # Adding field 'Application.ratings_average'
+        db.add_column('webcatalog_application', 'ratings_average', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=3, decimal_places=2, blank=True), keep_default=False)
+
+        # Adding field 'Application.ratings_histogram'
+        db.add_column('webcatalog_application', 'ratings_histogram', self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'ReviewStatsImport'
+        db.delete_table('webcatalog_reviewstatsimport')
+
+        # Deleting field 'Application.ratings_total'
+        db.delete_column('webcatalog_application', 'ratings_total')
+
+        # Deleting field 'Application.ratings_average'
+        db.delete_column('webcatalog_application', 'ratings_average')
+
+        # Deleting field 'Application.ratings_histogram'
+        db.delete_column('webcatalog_application', 'ratings_histogram')
+
+
+    models = {
+        'webcatalog.application': {
+            'Meta': {'unique_together': "(('distroseries', 'archive_id'),)", 'object_name': 'Application'},
+            'app_type': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}),
+            'architectures': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'archive_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '64', 'null': 'True', 'blank': 'True'}),
+            'categories': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'channel': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'comment': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'departments': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['webcatalog.Department']", 'symmetrical': 'False', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'distroseries': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['webcatalog.DistroSeries']"}),
+            'for_purchase': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'icon': ('django.db.models.fields.files.ImageField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'icon_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'keywords': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'mimetype': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'package_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'popcon': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'price': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '7', 'decimal_places': '2'}),
+            'ratings_average': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '3', 'decimal_places': '2', 'blank': 'True'}),
+            'ratings_histogram': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
+            'ratings_total': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'screenshot_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+            'section': ('django.db.models.fields.CharField', [], {'max_length': '32'})
+        },
+        'webcatalog.department': {
+            'Meta': {'object_name': 'Department'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['webcatalog.Department']", 'null': 'True', 'blank': 'True'})
+        },
+        'webcatalog.distroseries': {
+            'Meta': {'object_name': 'DistroSeries'},
+            'code_name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '10', 'blank': 'True'})
+        },
+        'webcatalog.reviewstatsimport': {
+            'Meta': {'object_name': 'ReviewStatsImport'},
+            'distroseries': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['webcatalog.DistroSeries']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_import': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'})
+        }
+    }
+
+    complete_apps = ['webcatalog']

=== modified file 'src/webcatalog/models/__init__.py'
--- src/webcatalog/models/__init__.py	2011-06-27 16:31:36 +0000
+++ src/webcatalog/models/__init__.py	2011-07-20 15:32:34 +0000
@@ -26,11 +26,13 @@
     'DistroSeries',
     'Application',
     'Department',
+    'ReviewStatsImport',
 ]
 
 from .oauthtoken import Token, Consumer, Nonce, DataStore
 from .applications import (
-    DistroSeries,
     Application,
     Department,
+    DistroSeries,
+    ReviewStatsImport,
 )

=== modified file 'src/webcatalog/models/applications.py'
--- src/webcatalog/models/applications.py	2011-07-07 20:54:45 +0000
+++ src/webcatalog/models/applications.py	2011-07-20 15:32:34 +0000
@@ -24,6 +24,7 @@
 
 import logging
 import re
+from datetime import datetime
 
 from django.core.urlresolvers import reverse
 from django.db import models
@@ -34,6 +35,7 @@
 __all__ = [
     'Application',
     'DistroSeries',
+    'ReviewStatsImport',
     ]
 
 
@@ -82,6 +84,11 @@
     price = models.DecimalField(max_digits=7, decimal_places=2, null=True,
         help_text="For-purchase applications (in US Dollars).")
 
+    ratings_total = models.IntegerField(null=True, blank=True)
+    ratings_average = models.DecimalField(max_digits=3, decimal_places=2,
+        null=True, blank=True)
+    ratings_histogram = models.CharField(max_length=128, blank=True)
+
     # Other desktop fields used by s-c
     # x-gnome-fullname
     # X-AppInstall-Ignore
@@ -171,3 +178,11 @@
 
     class Meta:
         app_label = 'webcatalog'
+
+
+class ReviewStatsImport(models.Model):
+    distroseries = models.ForeignKey(DistroSeries, unique=True)
+    last_import = models.DateTimeField(default=datetime.utcnow)
+
+    class Meta:
+        app_label = 'webcatalog'

=== modified file 'src/webcatalog/schema.py'
--- src/webcatalog/schema.py	2011-07-05 16:09:31 +0000
+++ src/webcatalog/schema.py	2011-07-20 15:32:34 +0000
@@ -60,3 +60,7 @@
     preflight = ConfigSection()
     preflight.preflight_base_template = StringConfigOption(
         default="webcatalog/base.html")
+
+    rnr = ConfigSection()
+    rnr.rnr_service_root = StringConfigOption(
+        default="http://reviews.ubuntu.com/reviews/api/1.0")

=== modified file 'src/webcatalog/tests/test_commands.py'
--- src/webcatalog/tests/test_commands.py	2011-07-06 14:12:57 +0000
+++ src/webcatalog/tests/test_commands.py	2011-07-20 15:32:34 +0000
@@ -25,6 +25,10 @@
 import os
 import shutil
 import tempfile
+from datetime import (
+    datetime,
+    timedelta,
+    )
 from decimal import Decimal
 
 from django.conf import settings
@@ -34,14 +38,19 @@
     MagicMock,
     Mock,
     )
+from rnrclient import ReviewsStats
 
 from webcatalog.models import (
     Application,
     DistroSeries,
+    ReviewStatsImport,
     )
 from webcatalog.management.commands.import_app_install_data import (
     Cache as CacheContextManager,
-    Command,
+    Command as ImportAppInstallCommand,
+    )
+from webcatalog.management.commands.import_ratings_stats import (
+    Command as ImportRatingsStatsCommand,
     )
 from webcatalog.tests.factory import TestCaseWithFactory
 
@@ -49,6 +58,7 @@
 __all__ = [
     'ImportAppInstallTestCase',
     'ImportForPurchaseAppsTestCase',
+    'ImportRatingsTestCase',
     ]
 
 
@@ -159,7 +169,7 @@
         with patch(get_uri_fn) as mock_get_uri:
             mock_get_uri.return_value = 'http://example.com/my.deb'
             with patch('urllib.urlretrieve') as mock_urlretrieve:
-                Command().get_latest_app_data_for_series('natty', tmp_dir)
+                ImportAppInstallCommand().get_latest_app_data_for_series('natty', tmp_dir)
         shutil.rmtree(tmp_dir)
 
         mock_urlretrieve.assert_called_with(
@@ -252,7 +262,7 @@
         self.assertTrue(scribus.icon.size > 1)
 
     def test_get_app_data_uri_for_series(self):
-        uri = Command().get_app_data_uri_for_series('natty')
+        uri = ImportAppInstallCommand().get_app_data_uri_for_series('natty')
 
         self.assertEqual('http://example.com/app-install-1.01.deb', uri)
 
@@ -342,7 +352,7 @@
             code_name='natty').count())
 
         with CacheContextManager('natty') as cache:
-            Command().update_from_cache(cache, 'natty')
+            ImportAppInstallCommand().update_from_cache(cache, 'natty')
 
         self.assertEqual(1, DistroSeries.objects.filter(
             code_name='natty').count())
@@ -351,7 +361,7 @@
         self.assertEqual(0, Application.objects.count())
 
         with CacheContextManager('natty') as cache:
-            Command().update_from_cache(cache, 'natty')
+            ImportAppInstallCommand().update_from_cache(cache, 'natty')
 
         # There are 4 packages in our mock_cache which is patched in the
         # setUp.
@@ -437,3 +447,84 @@
         app = Application.objects.get(name='MyApp',
             distroseries=self.distroseries)
         self.assertEqual(6461, app.icon.size)
+
+
+class ImportRatingsTestCase(TestCaseWithFactory):
+
+    def setUp(self):
+        super(ImportRatingsTestCase, self).setUp()
+        review_stats_fn = 'rnrclient.RatingsAndReviewsAPI.review_stats'
+        self.ratings_stats_patcher = patch(review_stats_fn)
+        self.mock_review_stats = self.ratings_stats_patcher.start()
+        self.mock_review_stats.return_value = []
+
+    def tearDown(self):
+        self.ratings_stats_patcher.stop()
+        super(ImportRatingsTestCase, self).tearDown()
+
+    def test_creates_last_import_record(self):
+        onion = self.factory.make_distroseries(code_name='onion')
+        self.assertEqual(0,
+            ReviewStatsImport.objects.filter(distroseries=onion).count())
+
+        call_command('import_ratings_stats', 'onion')
+
+        self.assertEqual(1,
+            ReviewStatsImport.objects.filter(distroseries=onion).count())
+
+    def test_updates_last_import_record(self):
+        onion = self.factory.make_distroseries(code_name='onion')
+        orig_timestamp = datetime(2011, 07, 18, 14, 43)
+        import_record = ReviewStatsImport.objects.create(
+            distroseries=onion, last_import=orig_timestamp)
+
+        call_command('import_ratings_stats', 'onion')
+
+        import_records = ReviewStatsImport.objects.filter(distroseries=onion)
+        self.assertEqual(1, import_records.count())
+        self.assertTrue(orig_timestamp < import_records[0].last_import)
+
+    def test_download_review_stats_no_previous(self):
+        # If there have been no previous imports, the complete stats
+        # will be retrieved.
+        onion = self.factory.make_distroseries(code_name='onion')
+
+        call_command('import_ratings_stats', 'onion')
+
+        self.mock_review_stats.assert_called_with(
+            origin='ubuntu', distroseries='onion')
+
+    def test_download_review_stats_with_recent(self):
+        # If there has been a recent import, we'll only grab the
+        # relevant stats.
+        onion = self.factory.make_distroseries(code_name='onion')
+        orig_timestamp = datetime.utcnow() - timedelta(2)
+        import_record = ReviewStatsImport.objects.create(
+            distroseries=onion, last_import=orig_timestamp)
+
+        call_command('import_ratings_stats', 'onion')
+
+        self.mock_review_stats.assert_called_with(
+            origin='ubuntu', distroseries='onion', days=3)
+
+    def test_update_app_stats(self):
+        # Any stats provided in the api call will be used to update our
+        # apps in the db.
+        natty = self.factory.make_distroseries(code_name='natty')
+        scribus = self.factory.make_application(package_name='scribus',
+            distroseries=natty)
+        otherapp = self.factory.make_application(package_name='otherapp',
+            distroseries=natty)
+
+        scribus_stats = ReviewsStats.from_dict(dict(package_name='scribus',
+            ratings_average='4.00', ratings_total=4,
+            histogram='[0, 1, 0, 1, 2]'))
+        self.mock_review_stats.return_value = [scribus_stats]
+        call_command('import_ratings_stats', 'natty')
+
+        scribus = Application.objects.get(id=scribus.id)
+        self.assertEqual(Decimal('4.00'), scribus.ratings_average)
+        self.assertEqual(4, scribus.ratings_total)
+        self.assertEqual('[0, 1, 0, 1, 2]', scribus.ratings_histogram)
+        otherapp = Application.objects.get(id=otherapp.id)
+        self.assertEqual(None, otherapp.ratings_total)