Merge lp:~michael.nelson/ubuntu-webcatalog/1019326-missing-ratings-summary into lp:ubuntu-webcatalog

[Michael Nelson]

Add admin class for ReviewStatsImport and optional --download-all flag for import_ratings_stats.

After some investigation on bug 1015604, this branch adds:

1) An admin class for ReviewStatsImport, so we can view/update when the last import occurred,
2) An --download-all flag for the import_ratings_stats command which forces all stats for the series to be downloaded, rather than requesting the optimised last X days,
3) An extra task so that we can request all ratings stats to be downloaded without needing a webop.

`fab test`

[Łukasz Czyżykowski (lukasz-czyzykowski)]

All great

review: Approve

=== modified file 'src/webcatalog/admin.py'
--- src/webcatalog/admin.py	2012-09-06 14:07:55 +0000
+++ src/webcatalog/admin.py	2012-12-03 17:04:22 +0000
@@ -28,6 +28,7 @@
     DistroSeries,
     Exhibit,
     Machine,
+    ReviewStatsImport,
 )
 
 __metaclass__ = type
@@ -80,6 +81,10 @@
     raw_id_fields = ('applications',)
 
 
+class ReviewStatsImportAdmin(admin.ModelAdmin):
+    list_display = ('distroseries', 'last_import')
+
+
 admin.site.register(Application, ApplicationAdmin)
 admin.site.register(ApplicationMedia, ApplicationMediaAdmin)
 admin.site.register(ApplicationWidget, ApplicationWidgetAdmin)
@@ -87,3 +92,4 @@
 admin.site.register(DistroSeries)
 admin.site.register(Exhibit, ExhibitAdmin)
 admin.site.register(Machine, MachineAdmin)
+admin.site.register(ReviewStatsImport, ReviewStatsImportAdmin)

=== modified file 'src/webcatalog/management/commands/import_all_ratings_stats.py'
--- src/webcatalog/management/commands/import_all_ratings_stats.py	2012-07-02 21:25:30 +0000
+++ src/webcatalog/management/commands/import_all_ratings_stats.py	2012-12-03 17:04:22 +0000
@@ -22,12 +22,23 @@
 __metaclass__ = type
 __all__ = []
 
+from optparse import make_option
+
 from django.conf import settings
 from django.core.management import call_command
 from django.core.management.base import NoArgsCommand
 
 
 class Command(NoArgsCommand):
+    option_list = NoArgsCommand.option_list + (
+        make_option(
+            '--download-all',
+            action='store_true',
+            dest='download_all',
+            default=False,
+            help='Download all ratings stats instead of stats since last '
+                 'import.'),
+    )
 
     def handle_noargs(self, **options):
         self.verbosity = int(options['verbosity'])
@@ -35,7 +46,8 @@
             self.output("Importing ratings stats for {0}\n".format(
                 distroseries), 1)
             call_command('import_ratings_stats', distroseries,
-                         verbosity=self.verbosity)
+                         verbosity=self.verbosity,
+                         download_all=options['download_all'])
 
     def output(self, message, level=None, flush=False):
         if hasattr(self, 'stdout'):

=== modified file 'src/webcatalog/management/commands/import_ratings_stats.py'
--- src/webcatalog/management/commands/import_ratings_stats.py	2012-06-19 13:10:25 +0000
+++ src/webcatalog/management/commands/import_ratings_stats.py	2012-12-03 17:04:22 +0000
@@ -17,11 +17,12 @@
 
 """Management command to import review statistics for a distroseries."""
 
+import json
 from datetime import (
     datetime,
     timedelta,
 )
-import json
+from optparse import make_option
 
 from django.core.management.base import LabelCommand
 
@@ -37,12 +38,22 @@
 
 
 class Command(LabelCommand):
+    option_list = LabelCommand.option_list + (
+        make_option(
+            '--download-all',
+            action='store_true',
+            dest='download_all',
+            default=False,
+            help='Download all ratings stats instead of stats since last '
+                 'import.'),
+    )
 
     help = "Update application review statistics."
     verbosity = 1
 
     def handle_label(self, distroseries_name, **options):
         self.verbosity = int(options['verbosity'])
+        self.download_all = options['download_all']
         # Check when we most recently imported review stats for this
         # distroseries.
         distroseries, created = DistroSeries.objects.get_or_create(
@@ -81,19 +92,20 @@
 
     def download_review_stats(self, distroseries):
         # Download the appropriate stats based on above.
-        other_imports = ReviewStatsImport.objects.filter(
-            distroseries=distroseries).order_by('-last_import')
         num_days = None
-        if other_imports:
-            stats_import = other_imports[0]
-            time_since_last_import = (
-                datetime.utcnow() - stats_import.last_import)
-            if time_since_last_import < timedelta(days=1):
-                num_days = 1
-            elif time_since_last_import < timedelta(days=3):
-                num_days = 3
-            elif time_since_last_import < timedelta(days=7):
-                num_days = 7
+        if not self.download_all:
+            other_imports = ReviewStatsImport.objects.filter(
+                distroseries=distroseries).order_by('-last_import')
+            if other_imports:
+                stats_import = other_imports[0]
+                time_since_last_import = (
+                    datetime.utcnow() - stats_import.last_import)
+                if time_since_last_import < timedelta(days=1):
+                    num_days = 1
+                elif time_since_last_import < timedelta(days=3):
+                    num_days = 3
+                elif time_since_last_import < timedelta(days=7):
+                    num_days = 7
 
         # Currently we have ratings in rnr for ubuntu and for-purchase
         # PPAs only, so we set origin to any. If we later support reviews

=== modified file 'src/webcatalog/tasks.py'
--- src/webcatalog/tasks.py	2012-06-29 19:51:06 +0000
+++ src/webcatalog/tasks.py	2012-12-03 17:04:22 +0000
@@ -25,10 +25,16 @@
 
 @task(name="webcatalog.tasks.import_ratings_stats")
 def import_ratings_stats():
-    """Import all ratings and reviews data"""
+    """Import ratings data since the last import"""
     call_command('import_all_ratings_stats', verbosity=0)
 
 
+@task(name="webcatalog.tasks.import_ratings_stats_all")
+def import_ratings_stats_all():
+    """Import all ratings data"""
+    call_command('import_all_ratings_stats', verbosity=0, download_all=True)
+
+
 @task(name="webcatalog.tasks.check_all_latest")
 def check_all_latest():
     """Update the 'is_latest' bit on all Applications"""

=== modified file 'src/webcatalog/tests/test_commands.py'
--- src/webcatalog/tests/test_commands.py	2012-08-29 21:46:54 +0000
+++ src/webcatalog/tests/test_commands.py	2012-12-03 17:04:22 +0000
@@ -38,6 +38,7 @@
 from django.test.testcases import TransactionTestCase
 
 from mock import (
+    call,
     patch,
     MagicMock,
     Mock,
@@ -773,6 +774,19 @@
         self.mock_review_stats.assert_called_with(
             origin='any', distroseries='onion', days=3)
 
+    def test_force_download_all_stats(self):
+        # If the download_all option is used, all stats are
+        # downloaded, rather than the optimised last X days.
+        onion = self.factory.make_distroseries(code_name='onion')
+        orig_timestamp = datetime.utcnow() - timedelta(2)
+        import_record = ReviewStatsImport.objects.create(
+            distroseries=onion, last_import=orig_timestamp)
+
+        call_command('import_ratings_stats', 'onion', download_all=True)
+
+        self.mock_review_stats.assert_called_with(origin='any',
+                                                  distroseries='onion')
+
     def test_update_app_stats(self):
         # Any stats provided in the api call will be used to update our
         # apps in the db.
@@ -1106,12 +1120,33 @@
 
 
 class ImportAllRatingsStatsTestCase(TestCaseWithFactory):
-    @patch('sys.stdout')
-    def test_handle(self, mock_stdout):
-        patch_attr = ('webcatalog.management.commands.'
-                      'import_all_ratings_stats.call_command')
-        with patch_settings(UBUNTU_SERIES_FOR_VERSIONS=TEST_VERSIONS):
-            with patch(patch_attr) as mock_call_command:
-                call_command('import_all_ratings_stats')
-
-            self.assertEqual(mock_call_command.call_count, len(TEST_VERSIONS))
+
+    def setUp(self):
+        patcher = patch('webcatalog.management.commands.'
+                        'import_all_ratings_stats.call_command')
+        self.mock_call_command = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        patcher = patch('sys.stdout')
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def test_handle_download(self):
+        with patch_settings(UBUNTU_SERIES_FOR_VERSIONS=TEST_VERSIONS):
+            call_command('import_all_ratings_stats')
+
+        expected_calls = [
+            call('import_ratings_stats', series, verbosity=1,
+                 download_all=False) for series in TEST_VERSIONS.values()]
+        self.mock_call_command.assert_has_calls(expected_calls,
+                                                any_order=True)
+
+    def test_handle_download_all(self):
+        with patch_settings(UBUNTU_SERIES_FOR_VERSIONS=TEST_VERSIONS):
+            call_command('import_all_ratings_stats', download_all=True)
+
+        expected_calls = [
+            call('import_ratings_stats', series, verbosity=1,
+                 download_all=True) for series in TEST_VERSIONS.values()]
+        self.mock_call_command.assert_has_calls(expected_calls,
+                                                any_order=True)

=== modified file 'src/webcatalog/tests/test_views.py'
--- src/webcatalog/tests/test_views.py	2012-09-06 14:07:55 +0000
+++ src/webcatalog/tests/test_views.py	2012-12-03 17:04:22 +0000
@@ -1394,10 +1394,14 @@
                 'name': 'import_sca',
             },
             {
-                'doc': 'Import all ratings and reviews data',
+                'doc': 'Import ratings data since the last import',
                 'name': 'import_ratings_stats',
             },
             {
+                'doc': 'Import all ratings data',
+                'name': 'import_ratings_stats_all',
+            },
+            {
                 'doc': "Update the 'is_latest' bit on all Applications",
                 'name': 'check_all_latest',
             },