Merge lp:~wgrant/launchpad/tm-performance-2-model into lp:launchpad

Proposed by William Grant on 2015-02-27
Status: Work in progress
Proposed branch: lp:~wgrant/launchpad/tm-performance-2-model
Merge into: lp:launchpad
Prerequisite: lp:~wgrant/launchpad/tm-performance-0-db
Diff against target: 498 lines (+175/-22)
10 files modified
lib/lp/testing/factory.py (+4/-2)
lib/lp/translations/interfaces/potemplate.py (+9/-0)
lib/lp/translations/interfaces/potmsgset.py (+3/-0)
lib/lp/translations/model/potemplate.py (+22/-3)
lib/lp/translations/model/potmsgset.py (+21/-11)
lib/lp/translations/model/translationmessage.py (+9/-0)
lib/lp/translations/model/translationtemplateitem.py (+4/-0)
lib/lp/translations/tests/test_potemplate.py (+41/-0)
lib/lp/translations/tests/test_potmsgset.py (+39/-3)
lib/lp/translations/tests/test_translationmessage.py (+23/-3)
To merge this branch: bzr merge lp:~wgrant/launchpad/tm-performance-2-model
Reviewer Review Type Date Requested Status
Launchpad code reviewers 2015-02-27 Pending
Review via email: mp+251223@code.launchpad.net
To post a comment you must log in.
17371. By William Grant on 2015-02-27

Set POTemplate.suggestive based on normal suggestiveness criteria, and add POTemplateSet.filterSuggestiveTargets with the same.

17372. By William Grant on 2015-02-27

Fix POTMsgSet.suggestive clone and tests.

17373. By William Grant on 2015-02-28

Merge tm-performance-0-db.

Unmerged revisions

17373. By William Grant on 2015-02-28

Merge tm-performance-0-db.

17372. By William Grant on 2015-02-27

Fix POTMsgSet.suggestive clone and tests.

17371. By William Grant on 2015-02-27

Set POTemplate.suggestive based on normal suggestiveness criteria, and add POTemplateSet.filterSuggestiveTargets with the same.

17370. By William Grant on 2015-02-27

Propagate suggestive down from POTemplate to POTMsgSet to TranslationMessage.

17369. By William Grant on 2015-02-27

submitSuggestion now uses _makeTranslationMessage.

17368. By William Grant on 2015-02-27

Set TranslationMessage.msgid_* from the POTMsgSet.

17367. By William Grant on 2015-02-27

Set TranslationTemplateItem.msgid_* from the POTMsgSet.

17366. By William Grant on 2015-02-27

Cached -> denormalised where appropriate.

17365. By William Grant on 2015-02-27

Add denormalised columns to the model.

17364. By William Grant on 2015-02-27

Merge tm-performance--1-disambiguate.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/testing/factory.py'
2--- lib/lp/testing/factory.py 2015-02-20 00:56:57 +0000
3+++ lib/lp/testing/factory.py 2015-02-28 00:25:08 +0000
4@@ -2923,7 +2923,7 @@
5 sourcepackagename=None, owner=None, name=None,
6 translation_domain=None, path=None,
7 copy_pofiles=True, side=None, sourcepackage=None,
8- iscurrent=True):
9+ iscurrent=True, suggestive=None):
10 """Make a new translation template."""
11 if sourcepackage is not None:
12 assert distroseries is None, (
13@@ -2964,7 +2964,9 @@
14 if path is None:
15 path = 'messages.pot'
16
17- pot = subset.new(name, translation_domain, path, owner, copy_pofiles)
18+ pot = subset.new(
19+ name, translation_domain, path, owner, copy_pofiles,
20+ suggestive)
21 removeSecurityProxy(pot).iscurrent = iscurrent
22 return pot
23
24
25=== modified file 'lib/lp/translations/interfaces/potemplate.py'
26--- lib/lp/translations/interfaces/potemplate.py 2014-07-01 16:01:39 +0000
27+++ lib/lp/translations/interfaces/potemplate.py 2015-02-28 00:25:08 +0000
28@@ -317,6 +317,9 @@
29 translation_side = Int(
30 title=_("Translation side"), required=True, readonly=True)
31
32+ suggestive = Bool(title=_(
33+ "Translations can provide suggestions to other templates (cached)."))
34+
35 def __iter__():
36 """Return an iterator over current `IPOTMsgSet` in this template."""
37
38@@ -719,6 +722,12 @@
39 :return: Number of rows inserted.
40 """
41
42+ def filterSuggestiveTargets(templates):
43+ """Return templates whose translations can be suggested to others.
44+
45+ This is cached as POTemplate.suggestive, but that can be out of date.
46+ """
47+
48
49 class IPOTemplateSharingSubset(Interface):
50 """A subset of sharing PO templates."""
51
52=== modified file 'lib/lp/translations/interfaces/potmsgset.py'
53--- lib/lp/translations/interfaces/potmsgset.py 2013-01-07 02:40:55 +0000
54+++ lib/lp/translations/interfaces/potmsgset.py 2015-02-28 00:25:08 +0000
55@@ -124,6 +124,9 @@
56 queries that search for credits messages.
57 """))
58
59+ suggestive = Bool(title=_(
60+ "Translations can provide suggestions to other templates (cached)."))
61+
62 def clone():
63 """Return a new copy of this POTMsgSet."""
64
65
66=== modified file 'lib/lp/translations/model/potemplate.py'
67--- lib/lp/translations/model/potemplate.py 2014-08-28 05:08:23 +0000
68+++ lib/lp/translations/model/potemplate.py 2015-02-28 00:25:08 +0000
69@@ -237,6 +237,9 @@
70 date_last_updated = UtcDateTimeCol(dbName='date_last_updated',
71 default=DEFAULT)
72
73+ # Cached.
74+ suggestive = BoolCol(dbName='suggestive', default=False)
75+
76 # joins
77 pofiles = SQLMultipleJoin('POFile', joinColumn='potemplate')
78
79@@ -849,7 +852,8 @@
80 commenttext=None,
81 filereferences=None,
82 sourcecomment=None,
83- flagscomment=None)
84+ flagscomment=None,
85+ suggestive=self.suggestive)
86
87 potmsgset.setSequence(self, sequence)
88 if potmsgset.is_translation_credit:
89@@ -1188,7 +1192,8 @@
90 """See `IPOTemplateSubset`."""
91 return self._getSuperSet().getPOTemplateByName(name) is None
92
93- def new(self, name, translation_domain, path, owner, copy_pofiles=True):
94+ def new(self, name, translation_domain, path, owner, copy_pofiles=True,
95+ suggestive=None):
96 """See `IPOTemplateSubset`."""
97 existing_template = self._getSuperSet().getPOTemplateByName(name)
98 if existing_template is not None:
99@@ -1201,6 +1206,10 @@
100 'languagename': 'LANGUAGE',
101 'languagecode': 'LL',
102 }
103+ if suggestive is None:
104+ series = self.productseries or self.distroseries
105+ suggestive = series.pillar.translations_usage in (
106+ ServiceUsage.LAUNCHPAD, ServiceUsage.EXTERNAL)
107 template = POTemplate(name=name,
108 translation_domain=translation_domain,
109 sourcepackagename=self.sourcepackagename,
110@@ -1208,7 +1217,8 @@
111 productseries=self.productseries,
112 path=path,
113 owner=owner,
114- header=standardTemplateHeader % header_params)
115+ header=standardTemplateHeader % header_params,
116+ suggestive=suggestive)
117 if copy_pofiles:
118 self._copyPOFilesFromSharingTemplates(template)
119 return template
120@@ -1380,10 +1390,19 @@
121 """See `IPOTemplateSet`."""
122 from lp.registry.model.product import Product
123 from lp.registry.model.productseries import ProductSeries
124+ templates = [removeSecurityProxy(t) for t in templates]
125 pses = load_related(ProductSeries, templates, ['productseriesID'])
126 load_related(Product, pses, ['productID'])
127 load_related(SourcePackageName, templates, ['sourcepackagenameID'])
128
129+ def filterSuggestiveTargets(self, templates):
130+ """See `IPOTemplateSet`."""
131+ return [
132+ template for template in templates
133+ if (template.product.translations_usage if template.product
134+ else template.distribution.translations_usage) in (
135+ ServiceUsage.LAUNCHPAD, ServiceUsage.EXTERNAL)]
136+
137 def wipeSuggestivePOTemplatesCache(self):
138 """See `IPOTemplateSet`."""
139 return IMasterStore(POTemplate).execute(
140
141=== modified file 'lib/lp/translations/model/potmsgset.py'
142--- lib/lp/translations/model/potmsgset.py 2015-02-27 05:47:55 +0000
143+++ lib/lp/translations/model/potmsgset.py 2015-02-28 00:25:08 +0000
144@@ -15,6 +15,7 @@
145 import re
146
147 from sqlobject import (
148+ BoolCol,
149 ForeignKey,
150 SQLObjectNotFound,
151 StringCol,
152@@ -143,6 +144,9 @@
153
154 credits_message_ids = credits_message_info.keys()
155
156+ # Cached from POTemplate.
157+ suggestive = BoolCol(dbName='suggestive', default=False)
158+
159 def clone(self):
160 return POTMsgSet(
161 context=self.context,
162@@ -152,6 +156,7 @@
163 filereferences=self.filereferences,
164 sourcecomment=self.sourcecomment,
165 flagscomment=self.flagscomment,
166+ suggestive=self.suggestive,
167 )
168
169 def _conflictsExistingSourceFileFormats(self, source_file_format=None):
170@@ -609,10 +614,6 @@
171 if existing_message is not None:
172 return existing_message
173
174- forms = dict(
175- ('msgstr%d' % form, potranslation)
176- for form, potranslation in potranslations.iteritems())
177-
178 if from_import:
179 origin = RosettaTranslationOrigin.SCM
180 else:
181@@ -620,10 +621,9 @@
182 pofile.potemplate.awardKarma(
183 submitter, 'translationsuggestionadded')
184
185- return TranslationMessage(
186- potmsgset=self, language=pofile.language,
187- origin=origin, submitter=submitter,
188- **forms)
189+ return self._makeTranslationMessage(
190+ pofile, submitter, potranslations, origin,
191+ validation_status=TranslationValidationStatus.UNKNOWN)
192
193 def _checkForConflict(self, current_message, lock_timestamp,
194 potranslations=None):
195@@ -733,7 +733,7 @@
196 return 'shared'
197
198 def _makeTranslationMessage(self, pofile, submitter, translations, origin,
199- diverged=False):
200+ diverged=False, validation_status=None):
201 """Create a new `TranslationMessage`.
202
203 The message will not be made current on either side (Ubuntu or
204@@ -741,6 +741,9 @@
205 current should be diverged, but it's up to the caller to ensure
206 the right state.
207 """
208+ if validation_status is None:
209+ validation_status = TranslationValidationStatus.OK
210+
211 if diverged:
212 potemplate = pofile.potemplate
213 else:
214@@ -750,7 +753,7 @@
215 ('msgstr%d' % form, translation)
216 for form, translation in translations.iteritems())
217
218- return TranslationMessage(
219+ tm = TranslationMessage(
220 potmsgset=self,
221 potemplate=potemplate,
222 pofile=pofile,
223@@ -758,7 +761,12 @@
224 origin=origin,
225 submitter=submitter,
226 validation_status=TranslationValidationStatus.OK,
227+ msgid_singular=self.msgid_singularID,
228+ msgid_plural=self.msgid_pluralID,
229+ suggestive=self.suggestive,
230 **translation_args)
231+ Store.of(tm).flush()
232+ return tm
233
234 def approveSuggestion(self, pofile, suggestion, reviewer,
235 share_with_other_side=False, lock_timestamp=None):
236@@ -1236,7 +1244,9 @@
237 return TranslationTemplateItem(
238 potemplate=potemplate,
239 sequence=sequence,
240- potmsgset=self)
241+ potmsgset=self,
242+ msgid_singular=self.msgid_singular,
243+ msgid_plural=self.msgid_plural)
244 else:
245 # There is no entry for this potmsgset in TranslationTemplateItem
246 # table, neither we need to create one, given that the sequence is
247
248=== modified file 'lib/lp/translations/model/translationmessage.py'
249--- lib/lp/translations/model/translationmessage.py 2014-07-10 05:30:29 +0000
250+++ lib/lp/translations/model/translationmessage.py 2015-02-28 00:25:08 +0000
251@@ -289,6 +289,13 @@
252 was_obsolete_in_last_import = BoolCol(
253 dbName='was_obsolete_in_last_import', notNull=True, default=False)
254
255+ # Denormalised from POTMsgSet.
256+ msgid_singular = ForeignKey(foreignKey='POMsgID', dbName='msgid_singular')
257+ msgid_plural = ForeignKey(foreignKey='POMsgID', dbName='msgid_plural')
258+
259+ # Cached.
260+ suggestive = BoolCol(dbName='suggestive', default=False)
261+
262 # XXX jamesh 2008-05-02:
263 # This method is not being called anymore. The Storm
264 # validator code doesn't handle getters.
265@@ -526,6 +533,8 @@
266 is_current_ubuntu=self.is_current_ubuntu,
267 is_current_upstream=self.is_current_upstream,
268 was_obsolete_in_last_import=self.was_obsolete_in_last_import,
269+ msgid_singular=self.msgid_singular,
270+ msgid_plural=self.msgid_plural, suggestive=self.suggestive,
271 )
272 return clone
273
274
275=== modified file 'lib/lp/translations/model/translationtemplateitem.py'
276--- lib/lp/translations/model/translationtemplateitem.py 2011-12-30 06:14:56 +0000
277+++ lib/lp/translations/model/translationtemplateitem.py 2015-02-28 00:25:08 +0000
278@@ -29,3 +29,7 @@
279 sequence = IntCol(dbName='sequence', notNull=True)
280 potmsgset = ForeignKey(
281 foreignKey='POTMsgSet', dbName='potmsgset', notNull=True)
282+
283+ # Denormalised from POTMsgSet.
284+ msgid_singular = ForeignKey(foreignKey='POMsgID', dbName='msgid_singular')
285+ msgid_plural = ForeignKey(foreignKey='POMsgID', dbName='msgid_plural')
286
287=== modified file 'lib/lp/translations/tests/test_potemplate.py'
288--- lib/lp/translations/tests/test_potemplate.py 2014-08-28 01:17:14 +0000
289+++ lib/lp/translations/tests/test_potemplate.py 2015-02-28 00:25:08 +0000
290@@ -12,6 +12,7 @@
291 from lp.registry.interfaces.distribution import IDistributionSet
292 from lp.services.worlddata.interfaces.language import ILanguageSet
293 from lp.testing import (
294+ admin_logged_in,
295 person_logged_in,
296 TestCaseWithFactory,
297 )
298@@ -1078,3 +1079,43 @@
299 productseries=self.factory.makeProductSeries())
300 self.assertEqual(
301 0, subset.getPOTemplatesByTranslationDomain("foo").count())
302+
303+
304+class TestPOTemplateSet(TestCaseWithFactory):
305+
306+ layer = DatabaseFunctionalLayer
307+
308+ def test_suggestiveness(self):
309+ # POTemplateSubset.new and POTemplateSet.filterSuggestiveTargets
310+ # say a template is able to make suggestions if its pillar has
311+ # LAUNCHPAD or EXTERNAL translations usage.
312+ with admin_logged_in():
313+ pg = self.factory.makeProduct()
314+ pg.translations_usage = ServiceUsage.EXTERNAL
315+ dg = self.factory.makeDistribution()
316+ dg.translations_usage = ServiceUsage.LAUNCHPAD
317+ self.factory.makeDistroSeries(distribution=dg)
318+ pb = self.factory.makeProduct()
319+ pb.translations_usage = ServiceUsage.UNKNOWN
320+ db = self.factory.makeDistribution()
321+ db.translations_usage = ServiceUsage.NOT_APPLICABLE
322+ self.factory.makeDistroSeries(distribution=db)
323+ spn = self.factory.makeSourcePackageName()
324+
325+ pots = {
326+ pg: self.factory.makePOTemplate(productseries=pg.series[0]),
327+ pb: self.factory.makePOTemplate(productseries=pb.series[0]),
328+ dg: self.factory.makePOTemplate(
329+ distroseries=dg.series[0], sourcepackagename=spn),
330+ db: self.factory.makePOTemplate(
331+ distroseries=db.series[0], sourcepackagename=spn),
332+ }
333+
334+ self.assertContentEqual(
335+ [pots[pg], pots[dg]],
336+ getUtility(IPOTemplateSet).filterSuggestiveTargets(
337+ pots.values()))
338+
339+ self.assertEqual(
340+ {pg: True, pb: False, dg: True, db: False},
341+ {pillar: pot.suggestive for pillar, pot in pots.iteritems()})
342
343=== modified file 'lib/lp/translations/tests/test_potmsgset.py'
344--- lib/lp/translations/tests/test_potmsgset.py 2014-01-30 15:04:06 +0000
345+++ lib/lp/translations/tests/test_potmsgset.py 2015-02-28 00:25:08 +0000
346@@ -9,12 +9,14 @@
347 )
348
349 import pytz
350+from testtools.matchers import MatchesStructure
351 import transaction
352 from zope.component import getUtility
353 from zope.security.proxy import removeSecurityProxy
354
355 from lp.app.enums import ServiceUsage
356 from lp.app.interfaces.launchpad import ILaunchpadCelebrities
357+from lp.services.database.interfaces import IStore
358 from lp.services.propertycache import get_property_cache
359 from lp.testing import TestCaseWithFactory
360 from lp.testing.layers import (
361@@ -38,6 +40,9 @@
362 TranslationConflict,
363 )
364 from lp.translations.model.translationmessage import DummyTranslationMessage
365+from lp.translations.model.translationtemplateitem import (
366+ TranslationTemplateItem,
367+ )
368
369
370 class TestTranslationSharedPOTMsgSets(TestCaseWithFactory):
371@@ -67,21 +72,51 @@
372 # Create a single POTMsgSet that is used across all tests,
373 # and add it to only one of the POTemplates.
374 self.potmsgset = self.factory.makePOTMsgSet(
375- self.devel_potemplate)
376+ self.devel_potemplate, singular=self.factory.getUniqueString(),
377+ plural=self.factory.getUniqueString())
378
379 def _refreshSuggestiveTemplatesCache(self):
380 """Refresh the `SuggestivePOTemplate` cache."""
381 getUtility(IPOTemplateSet).populateSuggestivePOTemplatesCache()
382
383+ def test_suggestive(self):
384+ # POTMsgSet.suggestive is set to the initial POTemplate's
385+ # suggestive value, and updated later by a garbo job.
386+ pot1 = self.factory.makePOTemplate(suggestive=False)
387+ pot2 = self.factory.makePOTemplate(suggestive=True)
388+
389+ # If created in a non-suggestive template, suggestive is false.
390+ potmsgset1 = self.factory.makePOTMsgSet(potemplate=pot1)
391+ self.assertFalse(potmsgset1.suggestive)
392+ potmsgset1.setSequence(pot2, 1)
393+ self.assertFalse(potmsgset1.suggestive)
394+
395+ # But when created in a suggestive template, suggestive is true.
396+ potmsgset2 = self.factory.makePOTMsgSet(potemplate=pot2)
397+ self.assertTrue(potmsgset2.suggestive)
398+
399 def test_TranslationTemplateItem(self):
400 self.potmsgset.setSequence(self.stable_potemplate, 1)
401
402 devel_potmsgsets = list(self.devel_potemplate.getPOTMsgSets())
403 stable_potmsgsets = list(self.stable_potemplate.getPOTMsgSets())
404
405+ # Both templates reference the same POTMsgSet.
406 self.assertEquals(devel_potmsgsets, [self.potmsgset])
407 self.assertEquals(devel_potmsgsets, stable_potmsgsets)
408
409+ # The msgids are denormalised from the POTMsgSet.
410+ ttis = IStore(TranslationTemplateItem).find(
411+ TranslationTemplateItem,
412+ TranslationTemplateItem.potemplateID.is_in(
413+ (self.devel_potemplate.id, self.stable_potemplate.id)))
414+ for tti in ttis:
415+ self.assertThat(
416+ tti,
417+ MatchesStructure.byEquality(
418+ msgid_singular=self.potmsgset.msgid_singular,
419+ msgid_plural=self.potmsgset.msgid_plural))
420+
421 def test_POTMsgSetInIncompatiblePOTemplates(self):
422 # Make sure a POTMsgSet cannot be used in two POTemplates with
423 # different incompatible source_file_format (like XPI and PO).
424@@ -1926,7 +1961,9 @@
425
426 def test_clone(self):
427 """Cloning a POTMsgSet should produce a near-identical copy."""
428+ pot = self.factory.makePOTemplate(suggestive=True)
429 msgset = self.factory.makePOTMsgSet(
430+ potemplate=pot,
431 context=self.factory.getUniqueString('context'),
432 plural=self.factory.getUniqueString('plural'),
433 singular=self.factory.getUniqueString('singular'),
434@@ -1936,8 +1973,6 @@
435 flagscomment=self.factory.getUniqueString('flagscomment'),
436 )
437 new_msgset = msgset.clone()
438- naked_msgset = removeSecurityProxy(msgset)
439- naked_new_msgset = removeSecurityProxy(new_msgset)
440 self.assertNotEqual(msgset.id, new_msgset.id)
441 self.assertEqual(msgset.context, new_msgset.context)
442 self.assertEqual(msgset.msgid_singular, new_msgset.msgid_singular)
443@@ -1947,3 +1982,4 @@
444 self.assertEqual(msgset.filereferences, new_msgset.filereferences)
445 self.assertEqual(msgset.sourcecomment, new_msgset.sourcecomment)
446 self.assertEqual(msgset.flagscomment, new_msgset.flagscomment)
447+ self.assertEqual(msgset.suggestive, new_msgset.suggestive)
448
449=== modified file 'lib/lp/translations/tests/test_translationmessage.py'
450--- lib/lp/translations/tests/test_translationmessage.py 2014-07-10 05:30:29 +0000
451+++ lib/lp/translations/tests/test_translationmessage.py 2015-02-28 00:25:08 +0000
452@@ -107,16 +107,33 @@
453 tm.potmsgset.setSequence(pofile.potemplate, 0)
454 self.assertEquals(None, tm.getOnePOFile())
455
456+ def test_denormalised(self):
457+ # msgid_singular and msgid_plural are denormalised from
458+ # POTMsgSet for suggestion performance.
459+ pot = self.factory.makePOTemplate(suggestive=True)
460+ singular = self.factory.getUniqueString()
461+ plural = self.factory.getUniqueString()
462+ potmsgset = self.factory.makePOTMsgSet(
463+ potemplate=pot, singular=singular, plural=plural)
464+ self.assertTrue(potmsgset.suggestive)
465+ tm = self.factory.makeCurrentTranslationMessage(
466+ potmsgset=potmsgset)
467+ naked_tm = removeSecurityProxy(tm)
468+ self.assertEqual(singular, naked_tm.msgid_singular.msgid)
469+ self.assertEqual(plural, naked_tm.msgid_plural.msgid)
470+ self.assertTrue(naked_tm.suggestive)
471+
472 def test_clone(self):
473 """Cloning a translation should produce a near-identical copy."""
474 translations = [self.factory.getUniqueString() for x in range(6)]
475- tm = self.factory.makeCurrentTranslationMessage(
476+ tm = removeSecurityProxy(self.factory.makeCurrentTranslationMessage(
477 date_created=self.factory.getUniqueDate(),
478- translations=translations, current_other=True)
479+ translations=translations, current_other=True))
480 tm.comment = self.factory.getUniqueString()
481 tm.was_obsolete_in_last_import = True
482+ tm.suggestive = True
483 potmsgset = self.factory.makePOTMsgSet()
484- clone = tm.clone(potmsgset)
485+ clone = removeSecurityProxy(tm.clone(potmsgset))
486 self.assertNotEqual(tm.id, clone.id)
487 self.assertIs(None, clone.potemplate)
488 self.assertEqual(potmsgset, clone.potmsgset)
489@@ -138,6 +155,9 @@
490 self.assertEqual(tm.is_current_upstream, clone.is_current_upstream)
491 self.assertEqual(
492 tm.was_obsolete_in_last_import, clone.was_obsolete_in_last_import)
493+ self.assertEqual(tm.msgid_singular, clone.msgid_singular)
494+ self.assertEqual(tm.msgid_plural, clone.msgid_plural)
495+ self.assertEqual(tm.suggestive, clone.suggestive)
496
497
498 class TestApprove(TestCaseWithFactory):