Merge lp:~wgrant/launchpad/tm-performance-2-model into lp:launchpad

[William Grant]

[lp:~wgrant/launchpad/tm-performance-2-model]

17371. By William Grant on 2015-02-27

Set POTemplate.suggestive based on normal suggestiveness criteria, and add POTemplateSet.filterSuggestiveTargets with the same.

17372. By William Grant on 2015-02-27

Fix POTMsgSet.suggestive clone and tests.

17373. By William Grant on 2015-02-28

Merge tm-performance-0-db.

Unmerged revisions

17373. By William Grant on 2015-02-28

Merge tm-performance-0-db.

17372. By William Grant on 2015-02-27

Fix POTMsgSet.suggestive clone and tests.

17371. By William Grant on 2015-02-27

Set POTemplate.suggestive based on normal suggestiveness criteria, and add POTemplateSet.filterSuggestiveTargets with the same.

17370. By William Grant on 2015-02-27

Propagate suggestive down from POTemplate to POTMsgSet to TranslationMessage.

17369. By William Grant on 2015-02-27

submitSuggestion now uses _makeTranslationMessage.

17368. By William Grant on 2015-02-27

Set TranslationMessage.msgid_* from the POTMsgSet.

17367. By William Grant on 2015-02-27

Set TranslationTemplateItem.msgid_* from the POTMsgSet.

17366. By William Grant on 2015-02-27

Cached -> denormalised where appropriate.

17365. By William Grant on 2015-02-27

Add denormalised columns to the model.

17364. By William Grant on 2015-02-27

Merge tm-performance--1-disambiguate.

=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py	2015-02-20 00:56:57 +0000
+++ lib/lp/testing/factory.py	2015-02-28 00:25:08 +0000
@@ -2923,7 +2923,7 @@
                        sourcepackagename=None, owner=None, name=None,
                        translation_domain=None, path=None,
                        copy_pofiles=True, side=None, sourcepackage=None,
-                       iscurrent=True):
+                       iscurrent=True, suggestive=None):
         """Make a new translation template."""
         if sourcepackage is not None:
             assert distroseries is None, (
@@ -2964,7 +2964,9 @@
         if path is None:
             path = 'messages.pot'
 
-        pot = subset.new(name, translation_domain, path, owner, copy_pofiles)
+        pot = subset.new(
+            name, translation_domain, path, owner, copy_pofiles,
+            suggestive)
         removeSecurityProxy(pot).iscurrent = iscurrent
         return pot
 
 
=== modified file 'lib/lp/translations/interfaces/potemplate.py'
--- lib/lp/translations/interfaces/potemplate.py	2014-07-01 16:01:39 +0000
+++ lib/lp/translations/interfaces/potemplate.py	2015-02-28 00:25:08 +0000
@@ -317,6 +317,9 @@
     translation_side = Int(
         title=_("Translation side"), required=True, readonly=True)
 
+    suggestive = Bool(title=_(
+        "Translations can provide suggestions to other templates (cached)."))
+
     def __iter__():
         """Return an iterator over current `IPOTMsgSet` in this template."""
 
@@ -719,6 +722,12 @@
         :return: Number of rows inserted.
         """
 
+    def filterSuggestiveTargets(templates):
+        """Return templates whose translations can be suggested to others.
+
+        This is cached as POTemplate.suggestive, but that can be out of date.
+        """
+
 
 class IPOTemplateSharingSubset(Interface):
     """A subset of sharing PO templates."""
 
=== modified file 'lib/lp/translations/interfaces/potmsgset.py'
--- lib/lp/translations/interfaces/potmsgset.py	2013-01-07 02:40:55 +0000
+++ lib/lp/translations/interfaces/potmsgset.py	2015-02-28 00:25:08 +0000
@@ -124,6 +124,9 @@
             queries that search for credits messages.
             """))
 
+    suggestive = Bool(title=_(
+        "Translations can provide suggestions to other templates (cached)."))
+
     def clone():
         """Return a new copy of this POTMsgSet."""
 
 
=== modified file 'lib/lp/translations/model/potemplate.py'
--- lib/lp/translations/model/potemplate.py	2014-08-28 05:08:23 +0000
+++ lib/lp/translations/model/potemplate.py	2015-02-28 00:25:08 +0000
@@ -237,6 +237,9 @@
     date_last_updated = UtcDateTimeCol(dbName='date_last_updated',
         default=DEFAULT)
 
+    # Cached.
+    suggestive = BoolCol(dbName='suggestive', default=False)
+
     # joins
     pofiles = SQLMultipleJoin('POFile', joinColumn='potemplate')
 
@@ -849,7 +852,8 @@
             commenttext=None,
             filereferences=None,
             sourcecomment=None,
-            flagscomment=None)
+            flagscomment=None,
+            suggestive=self.suggestive)
 
         potmsgset.setSequence(self, sequence)
         if potmsgset.is_translation_credit:
@@ -1188,7 +1192,8 @@
         """See `IPOTemplateSubset`."""
         return self._getSuperSet().getPOTemplateByName(name) is None
 
-    def new(self, name, translation_domain, path, owner, copy_pofiles=True):
+    def new(self, name, translation_domain, path, owner, copy_pofiles=True,
+            suggestive=None):
         """See `IPOTemplateSubset`."""
         existing_template = self._getSuperSet().getPOTemplateByName(name)
         if existing_template is not None:
@@ -1201,6 +1206,10 @@
             'languagename': 'LANGUAGE',
             'languagecode': 'LL',
             }
+        if suggestive is None:
+            series = self.productseries or self.distroseries
+            suggestive = series.pillar.translations_usage in (
+                ServiceUsage.LAUNCHPAD, ServiceUsage.EXTERNAL)
         template = POTemplate(name=name,
                           translation_domain=translation_domain,
                           sourcepackagename=self.sourcepackagename,
@@ -1208,7 +1217,8 @@
                           productseries=self.productseries,
                           path=path,
                           owner=owner,
-                          header=standardTemplateHeader % header_params)
+                          header=standardTemplateHeader % header_params,
+                          suggestive=suggestive)
         if copy_pofiles:
             self._copyPOFilesFromSharingTemplates(template)
         return template
@@ -1380,10 +1390,19 @@
         """See `IPOTemplateSet`."""
         from lp.registry.model.product import Product
         from lp.registry.model.productseries import ProductSeries
+        templates = [removeSecurityProxy(t) for t in templates]
         pses = load_related(ProductSeries, templates, ['productseriesID'])
         load_related(Product, pses, ['productID'])
         load_related(SourcePackageName, templates, ['sourcepackagenameID'])
 
+    def filterSuggestiveTargets(self, templates):
+        """See `IPOTemplateSet`."""
+        return [
+            template for template in templates
+            if (template.product.translations_usage if template.product
+                else template.distribution.translations_usage) in (
+                    ServiceUsage.LAUNCHPAD, ServiceUsage.EXTERNAL)]
+
     def wipeSuggestivePOTemplatesCache(self):
         """See `IPOTemplateSet`."""
         return IMasterStore(POTemplate).execute(
 
=== modified file 'lib/lp/translations/model/potmsgset.py'
--- lib/lp/translations/model/potmsgset.py	2015-02-27 05:47:55 +0000
+++ lib/lp/translations/model/potmsgset.py	2015-02-28 00:25:08 +0000
@@ -15,6 +15,7 @@
 import re
 
 from sqlobject import (
+    BoolCol,
     ForeignKey,
     SQLObjectNotFound,
     StringCol,
@@ -143,6 +144,9 @@
 
     credits_message_ids = credits_message_info.keys()
 
+    # Cached from POTemplate.
+    suggestive = BoolCol(dbName='suggestive', default=False)
+
     def clone(self):
         return POTMsgSet(
             context=self.context,
@@ -152,6 +156,7 @@
             filereferences=self.filereferences,
             sourcecomment=self.sourcecomment,
             flagscomment=self.flagscomment,
+            suggestive=self.suggestive,
         )
 
     def _conflictsExistingSourceFileFormats(self, source_file_format=None):
@@ -609,10 +614,6 @@
         if existing_message is not None:
             return existing_message
 
-        forms = dict(
-            ('msgstr%d' % form, potranslation)
-            for form, potranslation in potranslations.iteritems())
-
         if from_import:
             origin = RosettaTranslationOrigin.SCM
         else:
@@ -620,10 +621,9 @@
             pofile.potemplate.awardKarma(
                 submitter, 'translationsuggestionadded')
 
-        return TranslationMessage(
-            potmsgset=self, language=pofile.language,
-            origin=origin, submitter=submitter,
-            **forms)
+        return self._makeTranslationMessage(
+            pofile, submitter, potranslations, origin,
+            validation_status=TranslationValidationStatus.UNKNOWN)
 
     def _checkForConflict(self, current_message, lock_timestamp,
                           potranslations=None):
@@ -733,7 +733,7 @@
             return 'shared'
 
     def _makeTranslationMessage(self, pofile, submitter, translations, origin,
-                                diverged=False):
+                                diverged=False, validation_status=None):
         """Create a new `TranslationMessage`.
 
         The message will not be made current on either side (Ubuntu or
@@ -741,6 +741,9 @@
         current should be diverged, but it's up to the caller to ensure
         the right state.
         """
+        if validation_status is None:
+            validation_status = TranslationValidationStatus.OK
+
         if diverged:
             potemplate = pofile.potemplate
         else:
@@ -750,7 +753,7 @@
             ('msgstr%d' % form, translation)
             for form, translation in translations.iteritems())
 
-        return TranslationMessage(
+        tm = TranslationMessage(
             potmsgset=self,
             potemplate=potemplate,
             pofile=pofile,
@@ -758,7 +761,12 @@
             origin=origin,
             submitter=submitter,
             validation_status=TranslationValidationStatus.OK,
+            msgid_singular=self.msgid_singularID,
+            msgid_plural=self.msgid_pluralID,
+            suggestive=self.suggestive,
             **translation_args)
+        Store.of(tm).flush()
+        return tm
 
     def approveSuggestion(self, pofile, suggestion, reviewer,
                           share_with_other_side=False, lock_timestamp=None):
@@ -1236,7 +1244,9 @@
             return TranslationTemplateItem(
                 potemplate=potemplate,
                 sequence=sequence,
-                potmsgset=self)
+                potmsgset=self,
+                msgid_singular=self.msgid_singular,
+                msgid_plural=self.msgid_plural)
         else:
             # There is no entry for this potmsgset in TranslationTemplateItem
             # table, neither we need to create one, given that the sequence is
 
=== modified file 'lib/lp/translations/model/translationmessage.py'
--- lib/lp/translations/model/translationmessage.py	2014-07-10 05:30:29 +0000
+++ lib/lp/translations/model/translationmessage.py	2015-02-28 00:25:08 +0000
@@ -289,6 +289,13 @@
     was_obsolete_in_last_import = BoolCol(
         dbName='was_obsolete_in_last_import', notNull=True, default=False)
 
+    # Denormalised from POTMsgSet.
+    msgid_singular = ForeignKey(foreignKey='POMsgID', dbName='msgid_singular')
+    msgid_plural = ForeignKey(foreignKey='POMsgID', dbName='msgid_plural')
+
+    # Cached.
+    suggestive = BoolCol(dbName='suggestive', default=False)
+
     # XXX jamesh 2008-05-02:
     # This method is not being called anymore.  The Storm
     # validator code doesn't handle getters.
@@ -526,6 +533,8 @@
             is_current_ubuntu=self.is_current_ubuntu,
             is_current_upstream=self.is_current_upstream,
             was_obsolete_in_last_import=self.was_obsolete_in_last_import,
+            msgid_singular=self.msgid_singular,
+            msgid_plural=self.msgid_plural, suggestive=self.suggestive,
             )
         return clone
 
 
=== modified file 'lib/lp/translations/model/translationtemplateitem.py'
--- lib/lp/translations/model/translationtemplateitem.py	2011-12-30 06:14:56 +0000
+++ lib/lp/translations/model/translationtemplateitem.py	2015-02-28 00:25:08 +0000
@@ -29,3 +29,7 @@
     sequence = IntCol(dbName='sequence', notNull=True)
     potmsgset = ForeignKey(
         foreignKey='POTMsgSet', dbName='potmsgset', notNull=True)
+
+    # Denormalised from POTMsgSet.
+    msgid_singular = ForeignKey(foreignKey='POMsgID', dbName='msgid_singular')
+    msgid_plural = ForeignKey(foreignKey='POMsgID', dbName='msgid_plural')
 
=== modified file 'lib/lp/translations/tests/test_potemplate.py'
--- lib/lp/translations/tests/test_potemplate.py	2014-08-28 01:17:14 +0000
+++ lib/lp/translations/tests/test_potemplate.py	2015-02-28 00:25:08 +0000
@@ -12,6 +12,7 @@
 from lp.registry.interfaces.distribution import IDistributionSet
 from lp.services.worlddata.interfaces.language import ILanguageSet
 from lp.testing import (
+    admin_logged_in,
     person_logged_in,
     TestCaseWithFactory,
     )
@@ -1078,3 +1079,43 @@
             productseries=self.factory.makeProductSeries())
         self.assertEqual(
             0, subset.getPOTemplatesByTranslationDomain("foo").count())
+
+
+class TestPOTemplateSet(TestCaseWithFactory):
+
+    layer = DatabaseFunctionalLayer
+
+    def test_suggestiveness(self):
+        # POTemplateSubset.new and POTemplateSet.filterSuggestiveTargets
+        # say a template is able to make suggestions if its pillar has
+        # LAUNCHPAD or EXTERNAL translations usage.
+        with admin_logged_in():
+            pg = self.factory.makeProduct()
+            pg.translations_usage = ServiceUsage.EXTERNAL
+            dg = self.factory.makeDistribution()
+            dg.translations_usage = ServiceUsage.LAUNCHPAD
+            self.factory.makeDistroSeries(distribution=dg)
+            pb = self.factory.makeProduct()
+            pb.translations_usage = ServiceUsage.UNKNOWN
+            db = self.factory.makeDistribution()
+            db.translations_usage = ServiceUsage.NOT_APPLICABLE
+            self.factory.makeDistroSeries(distribution=db)
+        spn = self.factory.makeSourcePackageName()
+
+        pots = {
+            pg: self.factory.makePOTemplate(productseries=pg.series[0]),
+            pb: self.factory.makePOTemplate(productseries=pb.series[0]),
+            dg: self.factory.makePOTemplate(
+                distroseries=dg.series[0], sourcepackagename=spn),
+            db: self.factory.makePOTemplate(
+                distroseries=db.series[0], sourcepackagename=spn),
+            }
+
+        self.assertContentEqual(
+            [pots[pg], pots[dg]],
+            getUtility(IPOTemplateSet).filterSuggestiveTargets(
+                pots.values()))
+
+        self.assertEqual(
+            {pg: True, pb: False, dg: True, db: False},
+            {pillar: pot.suggestive for pillar, pot in pots.iteritems()})
 
=== modified file 'lib/lp/translations/tests/test_potmsgset.py'
--- lib/lp/translations/tests/test_potmsgset.py	2014-01-30 15:04:06 +0000
+++ lib/lp/translations/tests/test_potmsgset.py	2015-02-28 00:25:08 +0000
@@ -9,12 +9,14 @@
     )
 
 import pytz
+from testtools.matchers import MatchesStructure
 import transaction
 from zope.component import getUtility
 from zope.security.proxy import removeSecurityProxy
 
 from lp.app.enums import ServiceUsage
 from lp.app.interfaces.launchpad import ILaunchpadCelebrities
+from lp.services.database.interfaces import IStore
 from lp.services.propertycache import get_property_cache
 from lp.testing import TestCaseWithFactory
 from lp.testing.layers import (
@@ -38,6 +40,9 @@
     TranslationConflict,
     )
 from lp.translations.model.translationmessage import DummyTranslationMessage
+from lp.translations.model.translationtemplateitem import (
+    TranslationTemplateItem,
+    )
 
 
 class TestTranslationSharedPOTMsgSets(TestCaseWithFactory):
@@ -67,21 +72,51 @@
         # Create a single POTMsgSet that is used across all tests,
         # and add it to only one of the POTemplates.
         self.potmsgset = self.factory.makePOTMsgSet(
-            self.devel_potemplate)
+            self.devel_potemplate, singular=self.factory.getUniqueString(),
+            plural=self.factory.getUniqueString())
 
     def _refreshSuggestiveTemplatesCache(self):
         """Refresh the `SuggestivePOTemplate` cache."""
         getUtility(IPOTemplateSet).populateSuggestivePOTemplatesCache()
 
+    def test_suggestive(self):
+        # POTMsgSet.suggestive is set to the initial POTemplate's
+        # suggestive value, and updated later by a garbo job.
+        pot1 = self.factory.makePOTemplate(suggestive=False)
+        pot2 = self.factory.makePOTemplate(suggestive=True)
+
+        # If created in a non-suggestive template, suggestive is false.
+        potmsgset1 = self.factory.makePOTMsgSet(potemplate=pot1)
+        self.assertFalse(potmsgset1.suggestive)
+        potmsgset1.setSequence(pot2, 1)
+        self.assertFalse(potmsgset1.suggestive)
+
+        # But when created in a suggestive template, suggestive is true.
+        potmsgset2 = self.factory.makePOTMsgSet(potemplate=pot2)
+        self.assertTrue(potmsgset2.suggestive)
+
     def test_TranslationTemplateItem(self):
         self.potmsgset.setSequence(self.stable_potemplate, 1)
 
         devel_potmsgsets = list(self.devel_potemplate.getPOTMsgSets())
         stable_potmsgsets = list(self.stable_potemplate.getPOTMsgSets())
 
+        # Both templates reference the same POTMsgSet.
         self.assertEquals(devel_potmsgsets, [self.potmsgset])
         self.assertEquals(devel_potmsgsets, stable_potmsgsets)
 
+        # The msgids are denormalised from the POTMsgSet.
+        ttis = IStore(TranslationTemplateItem).find(
+            TranslationTemplateItem,
+            TranslationTemplateItem.potemplateID.is_in(
+                (self.devel_potemplate.id, self.stable_potemplate.id)))
+        for tti in ttis:
+            self.assertThat(
+                tti,
+                MatchesStructure.byEquality(
+                    msgid_singular=self.potmsgset.msgid_singular,
+                    msgid_plural=self.potmsgset.msgid_plural))
+
     def test_POTMsgSetInIncompatiblePOTemplates(self):
         # Make sure a POTMsgSet cannot be used in two POTemplates with
         # different incompatible source_file_format (like XPI and PO).
@@ -1926,7 +1961,9 @@
 
     def test_clone(self):
         """Cloning a POTMsgSet should produce a near-identical copy."""
+        pot = self.factory.makePOTemplate(suggestive=True)
         msgset = self.factory.makePOTMsgSet(
+            potemplate=pot,
             context=self.factory.getUniqueString('context'),
             plural=self.factory.getUniqueString('plural'),
             singular=self.factory.getUniqueString('singular'),
@@ -1936,8 +1973,6 @@
             flagscomment=self.factory.getUniqueString('flagscomment'),
         )
         new_msgset = msgset.clone()
-        naked_msgset = removeSecurityProxy(msgset)
-        naked_new_msgset = removeSecurityProxy(new_msgset)
         self.assertNotEqual(msgset.id, new_msgset.id)
         self.assertEqual(msgset.context, new_msgset.context)
         self.assertEqual(msgset.msgid_singular, new_msgset.msgid_singular)
@@ -1947,3 +1982,4 @@
         self.assertEqual(msgset.filereferences, new_msgset.filereferences)
         self.assertEqual(msgset.sourcecomment, new_msgset.sourcecomment)
         self.assertEqual(msgset.flagscomment, new_msgset.flagscomment)
+        self.assertEqual(msgset.suggestive, new_msgset.suggestive)
 
=== modified file 'lib/lp/translations/tests/test_translationmessage.py'
--- lib/lp/translations/tests/test_translationmessage.py	2014-07-10 05:30:29 +0000
+++ lib/lp/translations/tests/test_translationmessage.py	2015-02-28 00:25:08 +0000
@@ -107,16 +107,33 @@
         tm.potmsgset.setSequence(pofile.potemplate, 0)
         self.assertEquals(None, tm.getOnePOFile())
 
+    def test_denormalised(self):
+        # msgid_singular and msgid_plural are denormalised from
+        # POTMsgSet for suggestion performance.
+        pot = self.factory.makePOTemplate(suggestive=True)
+        singular = self.factory.getUniqueString()
+        plural = self.factory.getUniqueString()
+        potmsgset = self.factory.makePOTMsgSet(
+            potemplate=pot, singular=singular, plural=plural)
+        self.assertTrue(potmsgset.suggestive)
+        tm = self.factory.makeCurrentTranslationMessage(
+            potmsgset=potmsgset)
+        naked_tm = removeSecurityProxy(tm)
+        self.assertEqual(singular, naked_tm.msgid_singular.msgid)
+        self.assertEqual(plural, naked_tm.msgid_plural.msgid)
+        self.assertTrue(naked_tm.suggestive)
+
     def test_clone(self):
         """Cloning a translation should produce a near-identical copy."""
         translations = [self.factory.getUniqueString() for x in range(6)]
-        tm = self.factory.makeCurrentTranslationMessage(
+        tm = removeSecurityProxy(self.factory.makeCurrentTranslationMessage(
             date_created=self.factory.getUniqueDate(),
-            translations=translations, current_other=True)
+            translations=translations, current_other=True))
         tm.comment = self.factory.getUniqueString()
         tm.was_obsolete_in_last_import = True
+        tm.suggestive = True
         potmsgset = self.factory.makePOTMsgSet()
-        clone = tm.clone(potmsgset)
+        clone = removeSecurityProxy(tm.clone(potmsgset))
         self.assertNotEqual(tm.id, clone.id)
         self.assertIs(None, clone.potemplate)
         self.assertEqual(potmsgset, clone.potmsgset)
@@ -138,6 +155,9 @@
         self.assertEqual(tm.is_current_upstream, clone.is_current_upstream)
         self.assertEqual(
             tm.was_obsolete_in_last_import, clone.was_obsolete_in_last_import)
+        self.assertEqual(tm.msgid_singular, clone.msgid_singular)
+        self.assertEqual(tm.msgid_plural, clone.msgid_plural)
+        self.assertEqual(tm.suggestive, clone.suggestive)
 
 
 class TestApprove(TestCaseWithFactory):