Merge lp:~jtv/launchpad/recife-pre-stats into lp:launchpad/db-devel

[Jeroen T. Vermeulen]

[r=henninge][ui=none][no-qa] Pre-cleanups for Recife POFile stats.

= Cleanups before Recife statistics calculation =

The Translations application is moving over to its new "Recife" data model this cycle. One thing that still needs an overhaul is calculation of translation statistics for POFiles.

I'm working on that in other branches, but here's a small branch with some cleanups that can go in separately. The biggest change is a test that I've cleaned out a bit so that it no longer needs unusual privileges (or costly Launchpad layer setup) and runs in both old-model and new-model statistics calculation.

No lint. To exercise the test,
{{{
./bin/test -vvc lp.translations.tests.test_pofile
}}}

Jeroen

[Henning Eggers (henninge)]

<henninge> jtv: very nice simplification of the test.
<henninge> jtv: I just don't see what you describe as "runs in both old-model and new-model statistics calculation."
<jtv> In all honesty, I did it to get the test to start up faster. :-)
<jtv> I clear both flags as an alternative to deleting the message.
<henninge> ah! so lines 148-151 could use a comment I think ;)
<henninge> jtv: but apart from that, r=me ;)
* jtv looks
<jtv> OK, I'll note that.
<henninge> thanks

review: Approve (code)

=== modified file 'lib/lp/translations/interfaces/pofile.py'
--- lib/lp/translations/interfaces/pofile.py	2010-11-21 20:46:19 +0000
+++ lib/lp/translations/interfaces/pofile.py	2010-11-29 08:58:19 +0000
@@ -386,11 +386,14 @@
         if the end of the table has been reached.
         """
 
-    def getPOFilesWithTranslationCredits():
-        """Get all POFiles with potential translation credits messages.
+    def getPOFilesWithTranslationCredits(untranslated=False):
+        """Get POFiles with potential translation credits messages.
 
         Returns a ResultSet of (POFile, POTMsgSet) tuples, ordered by
         POFile.id.
+
+        :param untranslated: Look only for `POFile`s with a credits
+            message that is not translated.
         """
 
     def getPOFilesTouchedSince(date):

=== modified file 'lib/lp/translations/model/pofile.py'
--- lib/lp/translations/model/pofile.py	2010-11-26 05:10:27 +0000
+++ lib/lp/translations/model/pofile.py	2010-11-29 08:58:19 +0000
@@ -846,8 +846,6 @@
         if new_header is None:
             return
 
-        # XXX sabdfl 2005-05-27 should we also differentiate between
-        # washeaderfuzzy and isheaderfuzzy?
         self.topcomment = new_header.comment
         self.header = new_header.getRawContent()
         self.fuzzyheader = new_header.is_fuzzy
@@ -1424,7 +1422,8 @@
             TranslationTemplateItem.potemplateID == POFile.potemplateID,
             POTMsgSet.id == TranslationTemplateItem.potmsgsetID,
             POTMsgSet.msgid_singular == POMsgID.id,
-            POMsgID.msgid.is_in(POTMsgSet.credits_message_ids)]
+            POMsgID.msgid.is_in(POTMsgSet.credits_message_ids),
+            ]
         if untranslated:
             message_select = Select(
                 True,

=== modified file 'lib/lp/translations/model/potemplate.py'
--- lib/lp/translations/model/potemplate.py	2010-11-21 20:46:19 +0000
+++ lib/lp/translations/model/potemplate.py	2010-11-29 08:58:19 +0000
@@ -533,6 +533,10 @@
         """See `IRosettaStats`."""
         return self.messagecount
 
+    def updateMessageCount(self):
+        """Update `self.messagecount`."""
+        self.messagecount = self.getPOTMsgSetsCount()
+
     def currentCount(self, language=None):
         """See `IRosettaStats`."""
         if language is None:
@@ -944,7 +948,7 @@
             flush_database_updates()
 
             # Update cached number of msgsets.
-            self.messagecount = self.getPOTMsgSetsCount()
+            self.updateMessageCount()
 
             # The upload affects the statistics for all translations of this
             # template.  Recalculate those as well.  This takes time and

=== modified file 'lib/lp/translations/tests/test_pofile.py'
--- lib/lp/translations/tests/test_pofile.py	2010-11-26 05:10:27 +0000
+++ lib/lp/translations/tests/test_pofile.py	2010-11-29 08:58:19 +0000
@@ -20,10 +20,7 @@
 from canonical.database.constants import UTC_NOW
 from canonical.launchpad.interfaces.launchpad import ILaunchpadCelebrities
 from canonical.launchpad.webapp.publisher import canonical_url
-from canonical.testing.layers import (
-    LaunchpadZopelessLayer,
-    ZopelessDatabaseLayer,
-    )
+from canonical.testing.layers import ZopelessDatabaseLayer
 from lp.app.enums import ServiceUsage
 from lp.testing import TestCaseWithFactory
 from lp.translations.interfaces.pofile import IPOFileSet
@@ -1405,7 +1402,7 @@
 class TestPOFileSet(TestCaseWithFactory):
     """Test PO file set methods."""
 
-    layer = LaunchpadZopelessLayer
+    layer = ZopelessDatabaseLayer
 
     def setUp(self):
         # Create a POFileSet to work with.
@@ -1643,47 +1640,35 @@
                 self.pofileset.getPOFilesWithTranslationCredits()))
 
     def test_getPOFilesWithTranslationCredits_untranslated(self):
-        # We need absolute DB access to be able to remove a translation
-        # message.
-        LaunchpadZopelessLayer.switchDbUser('postgres')
+        # With "untranslated=True," getPOFilesWithTranslationCredits
+        # looks for POFiles whose translation credits messages are
+        # untranslated.
 
-        # Initially, we only get data from the sampledata, all of which
-        # are untranslated.
-        sampledata_pofiles = list(
+        # The sample data may contain some matching POFiles, but we'll
+        # ignore those.
+        initial_matches = set(
             self.pofileset.getPOFilesWithTranslationCredits(
                 untranslated=True))
-        total = len(sampledata_pofiles)
-        self.assertEquals(3, total)
 
-        # All POFiles with translation credits messages are
-        # returned along with relevant POTMsgSets.
-        potemplate1 = self.factory.makePOTemplate()
+        potemplate = self.factory.makePOTemplate()
         credits_potmsgset = self.factory.makePOTMsgSet(
-            potemplate1, singular=u'translator-credits', sequence=1)
-
-        sr_pofile = self.factory.makePOFile('sr', potemplate=potemplate1)
-        pofiles_with_credits = (
-            self.pofileset.getPOFilesWithTranslationCredits(
-                untranslated=True))
-        self.assertNotIn((sr_pofile, credits_potmsgset),
-                         list(pofiles_with_credits))
-        self.assertEquals(
-            total,
-            pofiles_with_credits.count())
-
-        # Removing a translation for this message, removes it
-        # from a result set when untranslated=True is passed in.
-        message = credits_potmsgset.getSharedTranslationMessage(
-            sr_pofile.language)
-        message.destroySelf()
-        pofiles_with_credits = (
-            self.pofileset.getPOFilesWithTranslationCredits(
-                untranslated=True))
-        self.assertIn((sr_pofile, credits_potmsgset),
-                      list(pofiles_with_credits))
-        self.assertEquals(
-            total + 1,
-            pofiles_with_credits.count())
+            potemplate, singular=u'translator-credits', sequence=1)
+        pofile = self.factory.makePOFile(potemplate=potemplate)
+
+        credits_translation = credits_potmsgset.getCurrentTranslation(
+            potemplate, pofile.language, potemplate.translation_side)
+        # Clearing is_current_upstream will make this an "untranslated"
+        # credits message.
+        # XXX JeroenVermeulen 2010-11-29: Also clearing
+        # is_current_ubuntu to make this pass until
+        # getPOFilesWithTranslationCredits is updated.
+        credits_translation.is_current_ubuntu = False
+        credits_translation.is_current_upstream = False
+
+        self.assertEqual(
+            initial_matches.union([(pofile, credits_potmsgset)]),
+            set(self.pofileset.getPOFilesWithTranslationCredits(
+                untranslated=True)))
 
     def test_getPOFilesByPathAndOrigin_path_mismatch(self):
         # getPOFilesByPathAndOrigin matches on POFile path.