Merge lp:~stevenk/launchpad/moar-preload-distroseries-queue into lp:launchpad

Proposed by Steve Kowalik
Status: Merged
Approved by: Steve Kowalik
Approved revision: no longer in the source branch.
Merged at revision: 16396
Proposed branch: lp:~stevenk/launchpad/moar-preload-distroseries-queue
Merge into: lp:launchpad
Diff against target: 568 lines (+139/-84)
11 files modified
lib/lp/security.py (+4/-6)
lib/lp/services/webapp/adapter.py (+0/-4)
lib/lp/soyuz/browser/queue.py (+19/-22)
lib/lp/soyuz/browser/tests/test_queue.py (+23/-2)
lib/lp/soyuz/configure.zcml (+2/-1)
lib/lp/soyuz/model/archive.py (+1/-1)
lib/lp/soyuz/model/binarypackagebuild.py (+1/-1)
lib/lp/soyuz/model/publishing.py (+12/-19)
lib/lp/soyuz/model/queue.py (+60/-21)
lib/lp/soyuz/model/sourcepackagerelease.py (+4/-7)
lib/lp/soyuz/tests/test_packageupload.py (+13/-0)
To merge this branch: bzr merge lp:~stevenk/launchpad/moar-preload-distroseries-queue
Reviewer Review Type Date Requested Status
William Grant code Approve
Review via email: mp+141581@code.launchpad.net

Commit message

Teach IPackageUploadSet.getAll() (and by extension the batching that drives DistroSeries:+queue) to do a lot more pre-loading.

Description of the change

Force the PackageUpload security adapter to use the cachedproperties on it, after making sure they are set carefully in add{Source,Build,Custom}.

Do lots of preloading in IPackageUploadSet.getAll() and in the preloading helper behind DistroSeries:+queue, like making ISourcePackageRelease.published_archives a cachedproperty, and populating it, and setting changes_file_id so we can preload all LFAs for changesfiles.

Write two query count tests, one for DistroSeries:+queue, and the other for IDistroSeries.getPackageUploads() (which is a thin wrapper around IPackageUploadSet.getAll())

To post a comment you must log in.
Revision history for this message
William Grant (wgrant) wrote :

143 + sprs = prefill_packageupload_caches(
144 + uploads, packageuploadsources, pubs, pucs)

Is this not already done by the underlying model method?

444 +def prefill_packageupload_caches(uploads, puses, pubs, pucs):

My eyes are burning.

Oh, god, they burn.

Can you add a bit of VWS, perhaps? There are several logical sections which could be separated.

468 + for spr in sprs:
469 + get_property_cache(spr).published_archives = []

The other three caches get away without this. Are they initialised elsewhere? Perhaps move them into this function as well.

471 + spr = get_property_cache(publication.sourcepackagerelease)

That's no SPR.

506 + @cachedproperty
507 + def published_archives(self):
508 + return list(self._published_archives())

There's no need for a separate _published_archives here, as nothing needs the actual ResultSet. Just inline it.

Additionally, you might want to invalidate this sometimes, though it's not used much so it's not hugely important. But IIRC SPPH creation is well encapsulated in PS.newSourcePublication,

review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/security.py'
--- lib/lp/security.py 2012-12-19 22:01:13 +0000
+++ lib/lp/security.py 2013-01-03 00:18:23 +0000
@@ -1,8 +1,6 @@
1# Copyright 2009-2012 Canonical Ltd. This software is licensed under the1# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4# pylint: disable-msg=F0401
5
6"""Security policies for using content objects."""4"""Security policies for using content objects."""
75
8__metaclass__ = type6__metaclass__ = type
@@ -1836,10 +1834,10 @@
1836 # We cannot use self.obj.sourcepackagerelease, as that causes1834 # We cannot use self.obj.sourcepackagerelease, as that causes
1837 # interference with the property cache if we are called in the1835 # interference with the property cache if we are called in the
1838 # process of adding a source or a build.1836 # process of adding a source or a build.
1839 if not self.obj._sources.is_empty():1837 if self.obj.sources:
1840 spr = self.obj._sources[0].sourcepackagerelease1838 spr = self.obj.sources[0].sourcepackagerelease
1841 elif not self.obj._builds.is_empty():1839 elif self.obj.builds:
1842 spr = self.obj._builds[0].build.source_package_release1840 spr = self.obj.builds[0].build.source_package_release
1843 else:1841 else:
1844 spr = None1842 spr = None
1845 if spr is not None:1843 if spr is not None:
18461844
=== modified file 'lib/lp/services/webapp/adapter.py'
--- lib/lp/services/webapp/adapter.py 2012-11-29 18:08:12 +0000
+++ lib/lp/services/webapp/adapter.py 2013-01-03 00:18:23 +0000
@@ -1,9 +1,6 @@
1# Copyright 2009-2011 Canonical Ltd. This software is licensed under the1# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4# We use global in this module.
5# pylint: disable-msg=W0602
6
7__metaclass__ = type4__metaclass__ = type
85
9from functools import partial6from functools import partial
@@ -442,7 +439,6 @@
442 for using connections from the main thread.439 for using connections from the main thread.
443 """440 """
444 # Record the ID of the main thread.441 # Record the ID of the main thread.
445 # pylint: disable-msg=W0603
446 global _main_thread_id442 global _main_thread_id
447 _main_thread_id = thread.get_ident()443 _main_thread_id = thread.get_ident()
448444
449445
=== modified file 'lib/lp/soyuz/browser/queue.py'
--- lib/lp/soyuz/browser/queue.py 2012-12-12 03:35:48 +0000
+++ lib/lp/soyuz/browser/queue.py 2013-01-03 00:18:23 +0000
@@ -10,7 +10,7 @@
10 'QueueItemsView',10 'QueueItemsView',
11 ]11 ]
1212
13import operator13from operator import attrgetter
1414
15from lazr.delegates import delegates15from lazr.delegates import delegates
16from zope.component import getUtility16from zope.component import getUtility
@@ -20,7 +20,7 @@
20 NotFoundError,20 NotFoundError,
21 UnexpectedFormData,21 UnexpectedFormData,
22 )22 )
23from lp.registry.model.person import Person23from lp.registry.interfaces.person import IPersonSet
24from lp.services.database.bulk import (24from lp.services.database.bulk import (
25 load_referencing,25 load_referencing,
26 load_related,26 load_related,
@@ -55,8 +55,10 @@
55 )55 )
56from lp.soyuz.interfaces.section import ISectionSet56from lp.soyuz.interfaces.section import ISectionSet
57from lp.soyuz.model.archive import Archive57from lp.soyuz.model.archive import Archive
58from lp.soyuz.model.component import Component
58from lp.soyuz.model.packagecopyjob import PackageCopyJob59from lp.soyuz.model.packagecopyjob import PackageCopyJob
59from lp.soyuz.model.queue import PackageUploadSource60from lp.soyuz.model.queue import PackageUploadSource
61from lp.soyuz.model.section import Section
60from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease62from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
6163
6264
@@ -124,8 +126,7 @@
124 build_ids = [binary_file.binarypackagerelease.build.id126 build_ids = [binary_file.binarypackagerelease.build.id
125 for binary_file in binary_files]127 for binary_file in binary_files]
126 upload_set = getUtility(IPackageUploadSet)128 upload_set = getUtility(IPackageUploadSet)
127 package_upload_builds = upload_set.getBuildByBuildIDs(129 package_upload_builds = upload_set.getBuildByBuildIDs(build_ids)
128 build_ids)
129 package_upload_builds_dict = {}130 package_upload_builds_dict = {}
130 for package_upload_build in package_upload_builds:131 for package_upload_build in package_upload_builds:
131 package_upload_builds_dict[132 package_upload_builds_dict[
@@ -135,8 +136,8 @@
135 def binary_files_dict(self, package_upload_builds_dict, binary_files):136 def binary_files_dict(self, package_upload_builds_dict, binary_files):
136 """Build a dictionary of lists of binary files keyed by upload ID.137 """Build a dictionary of lists of binary files keyed by upload ID.
137138
138 To do this efficiently we need to get all the PacakgeUploadBuild139 To do this efficiently we need to get all the PackageUploadBuild
139 records at once, otherwise the Ibuild.package_upload property140 records at once, otherwise the IBuild.package_upload property
140 causes one query per iteration of the loop.141 causes one query per iteration of the loop.
141 """142 """
142 build_upload_files = {}143 build_upload_files = {}
@@ -208,7 +209,9 @@
208 PackageCopyJob, uploads, ['package_copy_job_id'])209 PackageCopyJob, uploads, ['package_copy_job_id'])
209 load_related(Archive, package_copy_jobs, ['source_archive_id'])210 load_related(Archive, package_copy_jobs, ['source_archive_id'])
210 jobs = load_related(Job, package_copy_jobs, ['job_id'])211 jobs = load_related(Job, package_copy_jobs, ['job_id'])
211 load_related(Person, jobs, ['requester_id'])212 person_ids = map(attrgetter('requester_id'), jobs)
213 list(getUtility(IPersonSet).getPrecachedPersonsFromIDs(
214 person_ids, need_validity=True))
212215
213 def decoratedQueueBatch(self):216 def decoratedQueueBatch(self):
214 """Return the current batch, converted to decorated objects.217 """Return the current batch, converted to decorated objects.
@@ -224,20 +227,21 @@
224227
225 upload_ids = [upload.id for upload in uploads]228 upload_ids = [upload.id for upload in uploads]
226 binary_file_set = getUtility(IBinaryPackageFileSet)229 binary_file_set = getUtility(IBinaryPackageFileSet)
227 binary_files = binary_file_set.getByPackageUploadIDs(upload_ids)230 binary_files = list(binary_file_set.getByPackageUploadIDs(upload_ids))
228 binary_file_set.loadLibraryFiles(binary_files)231 binary_file_set.loadLibraryFiles(binary_files)
229 packageuploadsources = load_referencing(232 packageuploadsources = load_referencing(
230 PackageUploadSource, uploads, ['packageuploadID'])233 PackageUploadSource, uploads, ['packageuploadID'])
231 source_file_set = getUtility(ISourcePackageReleaseFileSet)234 source_file_set = getUtility(ISourcePackageReleaseFileSet)
232 source_files = source_file_set.getByPackageUploadIDs(upload_ids)235 source_files = list(source_file_set.getByPackageUploadIDs(upload_ids))
233
234 source_sprs = load_related(236 source_sprs = load_related(
235 SourcePackageRelease, packageuploadsources,237 SourcePackageRelease, packageuploadsources,
236 ['sourcepackagereleaseID'])238 ['sourcepackagereleaseID'])
237239
240 load_related(Section, source_sprs, ['sectionID'])
241 load_related(Component, source_sprs, ['componentID'])
242
238 # Get a dictionary of lists of binary files keyed by upload ID.243 # Get a dictionary of lists of binary files keyed by upload ID.
239 package_upload_builds_dict = self.builds_dict(244 package_upload_builds_dict = self.builds_dict(upload_ids, binary_files)
240 upload_ids, binary_files)
241245
242 build_upload_files, binary_package_names = self.binary_files_dict(246 build_upload_files, binary_package_names = self.binary_files_dict(
243 package_upload_builds_dict, binary_files)247 package_upload_builds_dict, binary_files)
@@ -461,7 +465,7 @@
461 sorted by their name.465 sorted by their name.
462 """466 """
463 return sorted(467 return sorted(
464 self.context.sections, key=operator.attrgetter('name'))468 self.context.sections, key=attrgetter('name'))
465469
466 def priorities(self):470 def priorities(self):
467 """An iterable of priorities from PackagePublishingPriority."""471 """An iterable of priorities from PackagePublishingPriority."""
@@ -516,8 +520,6 @@
516520
517 if self.contains_source:521 if self.contains_source:
518 self.sourcepackagerelease = self.sources[0].sourcepackagerelease522 self.sourcepackagerelease = self.sources[0].sourcepackagerelease
519
520 if self.contains_source:
521 self.package_sets = package_sets.get(523 self.package_sets = package_sets.get(
522 self.sourcepackagerelease.sourcepackagenameID, [])524 self.sourcepackagerelease.sourcepackagenameID, [])
523 else:525 else:
@@ -561,8 +563,7 @@
561 if title is None:563 if title is None:
562 title = alt564 title = alt
563 return structured(565 return structured(
564 '<img alt="[%s]" src="/@@/%s" title="%s" />',566 '<img alt="[%s]" src="/@@/%s" title="%s" />', alt, icon, title)
565 alt, icon, title)
566567
567 def composeIconList(self):568 def composeIconList(self):
568 """List icons that should be shown for this upload."""569 """List icons that should be shown for this upload."""
@@ -599,9 +600,5 @@
599 icon_string = structured('\n'.join(['%s'] * len(icons)), *icons)600 icon_string = structured('\n'.join(['%s'] * len(icons)), *icons)
600 link = self.composeNameAndChangesLink()601 link = self.composeNameAndChangesLink()
601 return structured(602 return structured(
602 """<div id="%s">603 """<div id="%s"> %s %s (%s)</div>""",
603 %s
604 %s
605 (%s)
606 </div>""",
607 iconlist_id, icon_string, link, self.displayarchs).escapedtext604 iconlist_id, icon_string, link, self.displayarchs).escapedtext
608605
=== modified file 'lib/lp/soyuz/browser/tests/test_queue.py'
--- lib/lp/soyuz/browser/tests/test_queue.py 2012-12-12 04:59:52 +0000
+++ lib/lp/soyuz/browser/tests/test_queue.py 2013-01-03 00:18:23 +0000
@@ -6,6 +6,8 @@
6__metaclass__ = type6__metaclass__ = type
77
8from lxml import html8from lxml import html
9from storm.store import Store
10from testtools.matchers import Equals
9import transaction11import transaction
10from zope.component import (12from zope.component import (
11 getUtility,13 getUtility,
@@ -26,12 +28,14 @@
26 login_person,28 login_person,
27 logout,29 logout,
28 person_logged_in,30 person_logged_in,
31 StormStatementRecorder,
29 TestCaseWithFactory,32 TestCaseWithFactory,
30 )33 )
31from lp.testing.layers import (34from lp.testing.layers import (
32 LaunchpadFunctionalLayer,35 LaunchpadFunctionalLayer,
33 LaunchpadZopelessLayer,36 LaunchpadZopelessLayer,
34 )37 )
38from lp.testing.matchers import HasQueryCount
35from lp.testing.sampledata import ADMIN_EMAIL39from lp.testing.sampledata import ADMIN_EMAIL
36from lp.testing.views import create_initialized_view40from lp.testing.views import create_initialized_view
3741
@@ -361,14 +365,31 @@
361 self.assertIn(365 self.assertIn(
362 upload.package_copy_job.job.requester.displayname, html_text)366 upload.package_copy_job.job.requester.displayname, html_text)
363367
368 def test_query_count(self):
369 login(ADMIN_EMAIL)
370 uploads = []
371 distroseries = self.factory.makeDistroSeries()
372 for i in range(5):
373 uploads.append(self.factory.makeSourcePackageUpload(distroseries))
374 uploads.append(self.factory.makeCustomPackageUpload(distroseries))
375 uploads.append(self.factory.makeCopyJobPackageUpload(distroseries))
376 for i in range(15):
377 uploads.append(self.factory.makeBuildPackageUpload(distroseries))
378 queue_admin = self.factory.makeArchiveAdmin(distroseries.main_archive)
379 Store.of(uploads[0]).invalidate()
380 with person_logged_in(queue_admin):
381 with StormStatementRecorder() as recorder:
382 view = self.makeView(distroseries, queue_admin)
383 view()
384 self.assertThat(recorder, HasQueryCount(Equals(52)))
385
364386
365class TestCompletePackageUpload(TestCaseWithFactory):387class TestCompletePackageUpload(TestCaseWithFactory):
366388
367 layer = LaunchpadZopelessLayer389 layer = LaunchpadZopelessLayer
368390
369 def makeCompletePackageUpload(self, upload=None, build_upload_files=None,391 def makeCompletePackageUpload(self, upload=None, build_upload_files=None,
370 source_upload_files=None,392 source_upload_files=None, package_sets=None):
371 package_sets=None):
372 if upload is None:393 if upload is None:
373 upload = self.factory.makeSourcePackageUpload()394 upload = self.factory.makeSourcePackageUpload()
374 if build_upload_files is None:395 if build_upload_files is None:
375396
=== modified file 'lib/lp/soyuz/configure.zcml'
--- lib/lp/soyuz/configure.zcml 2012-12-10 06:27:12 +0000
+++ lib/lp/soyuz/configure.zcml 2013-01-03 00:18:23 +0000
@@ -192,7 +192,8 @@
192 section_name192 section_name
193 components193 components
194 searchable_names194 searchable_names
195 searchable_versions"/>195 searchable_versions
196 changes_file_id"/>
196 <require197 <require
197 permission="launchpad.Edit"198 permission="launchpad.Edit"
198 attributes="199 attributes="
199200
=== modified file 'lib/lp/soyuz/model/archive.py'
--- lib/lp/soyuz/model/archive.py 2012-11-13 13:43:31 +0000
+++ lib/lp/soyuz/model/archive.py 2013-01-03 00:18:23 +0000
@@ -1554,7 +1554,7 @@
1554 PackageUploadSource.sourcepackagereleaseID,1554 PackageUploadSource.sourcepackagereleaseID,
1555 PackageUploadSource.packageuploadID == PackageUpload.id,1555 PackageUploadSource.packageuploadID == PackageUpload.id,
1556 PackageUpload.status == PackageUploadStatus.DONE,1556 PackageUpload.status == PackageUploadStatus.DONE,
1557 PackageUpload.changesfileID == LibraryFileAlias.id,1557 PackageUpload.changes_file_id == LibraryFileAlias.id,
1558 )1558 )
1559 else:1559 else:
1560 raise NotFoundError(filename)1560 raise NotFoundError(filename)
15611561
=== modified file 'lib/lp/soyuz/model/binarypackagebuild.py'
--- lib/lp/soyuz/model/binarypackagebuild.py 2012-11-15 01:42:33 +0000
+++ lib/lp/soyuz/model/binarypackagebuild.py 2013-01-03 00:18:23 +0000
@@ -180,7 +180,7 @@
180 Join(PackageUpload,180 Join(PackageUpload,
181 PackageUploadBuild.packageuploadID == PackageUpload.id),181 PackageUploadBuild.packageuploadID == PackageUpload.id),
182 Join(LibraryFileAlias,182 Join(LibraryFileAlias,
183 LibraryFileAlias.id == PackageUpload.changesfileID),183 LibraryFileAlias.id == PackageUpload.changes_file_id),
184 Join(LibraryFileContent,184 Join(LibraryFileContent,
185 LibraryFileContent.id == LibraryFileAlias.contentID),185 LibraryFileContent.id == LibraryFileAlias.contentID),
186 ]186 ]
187187
=== modified file 'lib/lp/soyuz/model/publishing.py'
--- lib/lp/soyuz/model/publishing.py 2012-11-15 23:28:13 +0000
+++ lib/lp/soyuz/model/publishing.py 2013-01-03 00:18:23 +0000
@@ -1,8 +1,6 @@
1# Copyright 2009-2012 Canonical Ltd. This software is licensed under the1# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4# pylint: disable-msg=E0611,W0212
5
6__metaclass__ = type4__metaclass__ = type
75
8__all__ = [6__all__ = [
@@ -1560,12 +1558,12 @@
1560 packageupload=packageupload)1558 packageupload=packageupload)
1561 DistributionSourcePackage.ensure(pub)1559 DistributionSourcePackage.ensure(pub)
15621560
1563 if create_dsd_job:1561 if create_dsd_job and archive == distroseries.main_archive:
1564 if archive == distroseries.main_archive:1562 dsd_job_source = getUtility(IDistroSeriesDifferenceJobSource)
1565 dsd_job_source = getUtility(IDistroSeriesDifferenceJobSource)1563 dsd_job_source.createForPackagePublication(
1566 dsd_job_source.createForPackagePublication(1564 distroseries, sourcepackagerelease.sourcepackagename, pocket)
1567 distroseries, sourcepackagerelease.sourcepackagename,1565 Store.of(sourcepackagerelease).flush()
1568 pocket)1566 del get_property_cache(sourcepackagerelease).published_archives
1569 return pub1567 return pub
15701568
1571 def getBuildsForSourceIds(self, source_publication_ids, archive=None,1569 def getBuildsForSourceIds(self, source_publication_ids, archive=None,
@@ -1802,8 +1800,7 @@
18021800
1803 return result_set1801 return result_set
18041802
1805 def getBinaryPublicationsForSources(self,1803 def getBinaryPublicationsForSources(self, one_or_more_source_publications):
1806 one_or_more_source_publications):
1807 """See `IPublishingSet`."""1804 """See `IPublishingSet`."""
1808 source_publication_ids = self._extractIDs(1805 source_publication_ids = self._extractIDs(
1809 one_or_more_source_publications)1806 one_or_more_source_publications)
@@ -1850,11 +1847,8 @@
18501847
1851 def getChangesFilesForSources(self, one_or_more_source_publications):1848 def getChangesFilesForSources(self, one_or_more_source_publications):
1852 """See `IPublishingSet`."""1849 """See `IPublishingSet`."""
1853 # Import PackageUpload and PackageUploadSource locally1850 # Avoid circular imports.
1854 # to avoid circular imports, since PackageUpload uses1851 from lp.soyuz.model.queue import PackageUpload, PackageUploadSource
1855 # SourcePackagePublishingHistory.
1856 from lp.soyuz.model.queue import (
1857 PackageUpload, PackageUploadSource)
18581852
1859 source_publication_ids = self._extractIDs(1853 source_publication_ids = self._extractIDs(
1860 one_or_more_source_publications)1854 one_or_more_source_publications)
@@ -1864,7 +1858,7 @@
1864 (SourcePackagePublishingHistory, PackageUpload,1858 (SourcePackagePublishingHistory, PackageUpload,
1865 SourcePackageRelease, LibraryFileAlias, LibraryFileContent),1859 SourcePackageRelease, LibraryFileAlias, LibraryFileContent),
1866 LibraryFileContent.id == LibraryFileAlias.contentID,1860 LibraryFileContent.id == LibraryFileAlias.contentID,
1867 LibraryFileAlias.id == PackageUpload.changesfileID,1861 LibraryFileAlias.id == PackageUpload.changes_file_id,
1868 PackageUpload.id == PackageUploadSource.packageuploadID,1862 PackageUpload.id == PackageUploadSource.packageuploadID,
1869 PackageUpload.status == PackageUploadStatus.DONE,1863 PackageUpload.status == PackageUploadStatus.DONE,
1870 PackageUpload.distroseriesID ==1864 PackageUpload.distroseriesID ==
@@ -1882,14 +1876,13 @@
18821876
1883 def getChangesFileLFA(self, spr):1877 def getChangesFileLFA(self, spr):
1884 """See `IPublishingSet`."""1878 """See `IPublishingSet`."""
1885 # Import PackageUpload and PackageUploadSource locally to avoid1879 # Avoid circular imports.
1886 # circular imports.
1887 from lp.soyuz.model.queue import PackageUpload, PackageUploadSource1880 from lp.soyuz.model.queue import PackageUpload, PackageUploadSource
18881881
1889 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)1882 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
1890 result_set = store.find(1883 result_set = store.find(
1891 LibraryFileAlias,1884 LibraryFileAlias,
1892 LibraryFileAlias.id == PackageUpload.changesfileID,1885 LibraryFileAlias.id == PackageUpload.changes_file_id,
1893 PackageUpload.status == PackageUploadStatus.DONE,1886 PackageUpload.status == PackageUploadStatus.DONE,
1894 PackageUpload.distroseriesID == spr.upload_distroseries.id,1887 PackageUpload.distroseriesID == spr.upload_distroseries.id,
1895 PackageUpload.archiveID == spr.upload_archive.id,1888 PackageUpload.archiveID == spr.upload_archive.id,
18961889
=== modified file 'lib/lp/soyuz/model/queue.py'
--- lib/lp/soyuz/model/queue.py 2012-12-26 01:04:05 +0000
+++ lib/lp/soyuz/model/queue.py 2013-01-03 00:18:23 +0000
@@ -52,7 +52,10 @@
52from lp.registry.model.sourcepackagename import SourcePackageName52from lp.registry.model.sourcepackagename import SourcePackageName
53from lp.services.auditor.client import AuditorClient53from lp.services.auditor.client import AuditorClient
54from lp.services.config import config54from lp.services.config import config
55from lp.services.database.bulk import load_referencing55from lp.services.database.bulk import (
56 load_referencing,
57 load_related,
58 )
56from lp.services.database.constants import UTC_NOW59from lp.services.database.constants import UTC_NOW
57from lp.services.database.datetimecol import UtcDateTimeCol60from lp.services.database.datetimecol import UtcDateTimeCol
58from lp.services.database.decoratedresultset import DecoratedResultSet61from lp.services.database.decoratedresultset import DecoratedResultSet
@@ -112,6 +115,7 @@
112 QueueStateWriteProtectedError,115 QueueStateWriteProtectedError,
113 )116 )
114from lp.soyuz.interfaces.section import ISectionSet117from lp.soyuz.interfaces.section import ISectionSet
118from lp.soyuz.model.distroarchseries import DistroArchSeries
115from lp.soyuz.pas import BuildDaemonPackagesArchSpecific119from lp.soyuz.pas import BuildDaemonPackagesArchSpecific
116120
117# There are imports below in PackageUploadCustom for various bits121# There are imports below in PackageUploadCustom for various bits
@@ -174,8 +178,8 @@
174 dbName='pocket', unique=False, notNull=True,178 dbName='pocket', unique=False, notNull=True,
175 schema=PackagePublishingPocket)179 schema=PackagePublishingPocket)
176180
177 changesfile = ForeignKey(181 changes_file_id = Int(name='changesfile')
178 dbName='changesfile', foreignKey="LibraryFileAlias", notNull=False)182 changesfile = Reference(changes_file_id, 'LibraryFileAlias.id')
179183
180 archive = ForeignKey(dbName="archive", foreignKey="Archive", notNull=True)184 archive = ForeignKey(dbName="archive", foreignKey="Archive", notNull=True)
181185
@@ -814,15 +818,16 @@
814818
815 def addSource(self, spr):819 def addSource(self, spr):
816 """See `IPackageUpload`."""820 """See `IPackageUpload`."""
817 del get_property_cache(self).sources
818 self.addSearchableNames([spr.name])821 self.addSearchableNames([spr.name])
819 self.addSearchableVersions([spr.version])822 self.addSearchableVersions([spr.version])
820 return PackageUploadSource(823 pus = PackageUploadSource(
821 packageupload=self, sourcepackagerelease=spr.id)824 packageupload=self, sourcepackagerelease=spr.id)
825 Store.of(self).flush()
826 del get_property_cache(self).sources
827 return pus
822828
823 def addBuild(self, build):829 def addBuild(self, build):
824 """See `IPackageUpload`."""830 """See `IPackageUpload`."""
825 del get_property_cache(self).builds
826 names = [build.source_package_release.name]831 names = [build.source_package_release.name]
827 versions = []832 versions = []
828 for bpr in build.binarypackages:833 for bpr in build.binarypackages:
@@ -830,15 +835,20 @@
830 versions.append(bpr.version)835 versions.append(bpr.version)
831 self.addSearchableNames(names)836 self.addSearchableNames(names)
832 self.addSearchableVersions(versions)837 self.addSearchableVersions(versions)
833 return PackageUploadBuild(packageupload=self, build=build.id)838 pub = PackageUploadBuild(packageupload=self, build=build.id)
839 Store.of(self).flush()
840 del get_property_cache(self).builds
841 return pub
834842
835 def addCustom(self, library_file, custom_type):843 def addCustom(self, library_file, custom_type):
836 """See `IPackageUpload`."""844 """See `IPackageUpload`."""
837 del get_property_cache(self).customfiles
838 self.addSearchableNames([library_file.filename])845 self.addSearchableNames([library_file.filename])
839 return PackageUploadCustom(846 puc = PackageUploadCustom(
840 packageupload=self, libraryfilealias=library_file.id,847 packageupload=self, libraryfilealias=library_file.id,
841 customformat=custom_type)848 customformat=custom_type)
849 Store.of(self).flush()
850 del get_property_cache(self).customfiles
851 return puc
842852
843 def isPPA(self):853 def isPPA(self):
844 """See `IPackageUpload`."""854 """See `IPackageUpload`."""
@@ -1665,18 +1675,7 @@
1665 pucs = load_referencing(1675 pucs = load_referencing(
1666 PackageUploadCustom, rows, ["packageuploadID"])1676 PackageUploadCustom, rows, ["packageuploadID"])
16671677
1668 for pu in rows:1678 prefill_packageupload_caches(rows, puses, pubs, pucs)
1669 cache = get_property_cache(pu)
1670 cache.sources = []
1671 cache.builds = []
1672 cache.customfiles = []
1673
1674 for pus in puses:
1675 get_property_cache(pus.packageupload).sources.append(pus)
1676 for pub in pubs:
1677 get_property_cache(pub.packageupload).builds.append(pub)
1678 for puc in pucs:
1679 get_property_cache(puc.packageupload).customfiles.append(puc)
16801679
1681 return DecoratedResultSet(query, pre_iter_hook=preload_hook)1680 return DecoratedResultSet(query, pre_iter_hook=preload_hook)
16821681
@@ -1704,3 +1703,43 @@
1704 return IStore(PackageUpload).find(1703 return IStore(PackageUpload).find(
1705 PackageUpload,1704 PackageUpload,
1706 PackageUpload.package_copy_job_id.is_in(pcj_ids))1705 PackageUpload.package_copy_job_id.is_in(pcj_ids))
1706
1707
1708def prefill_packageupload_caches(uploads, puses, pubs, pucs):
1709 # Circular imports.
1710 from lp.soyuz.model.archive import Archive
1711 from lp.soyuz.model.binarypackagebuild import BinaryPackageBuild
1712 from lp.soyuz.model.publishing import SourcePackagePublishingHistory
1713 from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
1714
1715 for pu in uploads:
1716 cache = get_property_cache(pu)
1717 cache.sources = []
1718 cache.builds = []
1719 cache.customfiles = []
1720
1721 for pus in puses:
1722 get_property_cache(pus.packageupload).sources.append(pus)
1723 for pub in pubs:
1724 get_property_cache(pub.packageupload).builds.append(pub)
1725 for puc in pucs:
1726 get_property_cache(puc.packageupload).customfiles.append(puc)
1727
1728 source_sprs = load_related(
1729 SourcePackageRelease, puses, ['sourcepackagereleaseID'])
1730 bpbs = load_related(BinaryPackageBuild, pubs, ['buildID'])
1731 load_related(DistroArchSeries, bpbs, ['distro_arch_series_id'])
1732 binary_sprs = load_related(
1733 SourcePackageRelease, bpbs, ['source_package_release_id'])
1734 sprs = source_sprs + binary_sprs
1735
1736 load_related(SourcePackageName, sprs, ['sourcepackagenameID'])
1737 load_related(LibraryFileAlias, uploads, ['changes_file_id'])
1738 publications = load_referencing(
1739 SourcePackagePublishingHistory, sprs, ['sourcepackagereleaseID'])
1740 load_related(Archive, publications, ['archiveID'])
1741 for spr_cache in sprs:
1742 get_property_cache(spr_cache).published_archives = []
1743 for publication in publications:
1744 spr_cache = get_property_cache(publication.sourcepackagerelease)
1745 spr_cache.published_archives.append(publication.archive)
17071746
=== modified file 'lib/lp/soyuz/model/sourcepackagerelease.py'
--- lib/lp/soyuz/model/sourcepackagerelease.py 2012-11-26 12:53:30 +0000
+++ lib/lp/soyuz/model/sourcepackagerelease.py 2013-01-03 00:18:23 +0000
@@ -1,8 +1,6 @@
1# Copyright 2009-2012 Canonical Ltd. This software is licensed under the1# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4# pylint: disable-msg=E0611,W0212
5
6__metaclass__ = type4__metaclass__ = type
7__all__ = [5__all__ = [
8 'SourcePackageRelease',6 'SourcePackageRelease',
@@ -291,14 +289,13 @@
291 @property289 @property
292 def current_publishings(self):290 def current_publishings(self):
293 """See ISourcePackageRelease."""291 """See ISourcePackageRelease."""
294 from lp.soyuz.model.distroseriessourcepackagerelease \292 from lp.soyuz.model.distroseriessourcepackagerelease import (
295 import DistroSeriesSourcePackageRelease293 DistroSeriesSourcePackageRelease)
296 return [DistroSeriesSourcePackageRelease(pub.distroseries, self)294 return [DistroSeriesSourcePackageRelease(pub.distroseries, self)
297 for pub in self.publishings]295 for pub in self.publishings]
298296
299 @property297 @cachedproperty
300 def published_archives(self):298 def published_archives(self):
301 """See `ISourcePackageRelease`."""
302 archives = set(299 archives = set(
303 pub.archive for pub in self.publishings.prejoin(['archive']))300 pub.archive for pub in self.publishings.prejoin(['archive']))
304 return sorted(archives, key=operator.attrgetter('id'))301 return sorted(archives, key=operator.attrgetter('id'))
@@ -503,7 +500,7 @@
503 Join(PackageUpload,500 Join(PackageUpload,
504 PackageUploadSource.packageuploadID == PackageUpload.id),501 PackageUploadSource.packageuploadID == PackageUpload.id),
505 Join(LibraryFileAlias,502 Join(LibraryFileAlias,
506 LibraryFileAlias.id == PackageUpload.changesfileID),503 LibraryFileAlias.id == PackageUpload.changes_file_id),
507 Join(LibraryFileContent,504 Join(LibraryFileContent,
508 LibraryFileContent.id == LibraryFileAlias.contentID),505 LibraryFileContent.id == LibraryFileAlias.contentID),
509 ]506 ]
510507
=== modified file 'lib/lp/soyuz/tests/test_packageupload.py'
--- lib/lp/soyuz/tests/test_packageupload.py 2012-12-17 05:10:29 +0000
+++ lib/lp/soyuz/tests/test_packageupload.py 2013-01-03 00:18:23 +0000
@@ -1292,3 +1292,16 @@
1292 "customformat": "raw-translations",1292 "customformat": "raw-translations",
1293 }1293 }
1294 self.assertEqual(expected_custom, ws_binaries[-1])1294 self.assertEqual(expected_custom, ws_binaries[-1])
1295
1296 def test_getPackageUploads_query_count(self):
1297 person = self.makeQueueAdmin([self.universe])
1298 uploads = []
1299 for i in range(5):
1300 upload, _ = self.makeBinaryPackageUpload(
1301 person, component=self.universe)
1302 uploads.append(upload)
1303 ws_distroseries = self.load(self.distroseries, person)
1304 IStore(uploads[0].__class__).invalidate()
1305 with StormStatementRecorder() as recorder:
1306 ws_distroseries.getPackageUploads()
1307 self.assertThat(recorder, HasQueryCount(Equals(27)))