Merge lp:~wgrant/launchpad/bug-728836 into lp:launchpad
- bug-728836
- Merge into devel
Proposed by
William Grant
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | Robert Collins | ||||
Approved revision: | no longer in the source branch. | ||||
Merged at revision: | 12537 | ||||
Proposed branch: | lp:~wgrant/launchpad/bug-728836 | ||||
Merge into: | lp:launchpad | ||||
Diff against target: |
574 lines (+286/-77) 9 files modified
lib/lp/registry/interfaces/distroseries.py (+1/-1) lib/lp/registry/model/distroseries.py (+2/-2) lib/lp/soyuz/interfaces/binarypackagerelease.py (+1/-0) lib/lp/soyuz/interfaces/publishing.py (+21/-4) lib/lp/soyuz/model/archive.py (+6/-1) lib/lp/soyuz/model/publishing.py (+139/-61) lib/lp/soyuz/model/queue.py (+3/-6) lib/lp/soyuz/tests/test_publishing.py (+111/-0) lib/lp/testing/factory.py (+2/-2) |
||||
To merge this branch: | bzr merge lp:~wgrant/launchpad/bug-728836 | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Robert Collins (community) | Approve | ||
Review via email: mp+52355@code.launchpad.net |
Commit message
[r=lifeless][bug=728836] Optimise PublishingSet.
Description of the change
This branch improves the scalability of Archive:
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/lp/registry/interfaces/distroseries.py' | |||
2 | --- lib/lp/registry/interfaces/distroseries.py 2011-02-24 15:30:54 +0000 | |||
3 | +++ lib/lp/registry/interfaces/distroseries.py 2011-03-07 07:05:26 +0000 | |||
4 | @@ -767,7 +767,7 @@ | |||
5 | 767 | """ | 767 | """ |
6 | 768 | 768 | ||
7 | 769 | def newArch(architecturetag, processorfamily, official, owner, | 769 | def newArch(architecturetag, processorfamily, official, owner, |
9 | 770 | supports_virtualized=False): | 770 | supports_virtualized=False, enabled=True): |
10 | 771 | """Create a new port or DistroArchSeries for this DistroSeries.""" | 771 | """Create a new port or DistroArchSeries for this DistroSeries.""" |
11 | 772 | 772 | ||
12 | 773 | def copyTranslationsFromParent(ztm): | 773 | def copyTranslationsFromParent(ztm): |
13 | 774 | 774 | ||
14 | === modified file 'lib/lp/registry/model/distroseries.py' | |||
15 | --- lib/lp/registry/model/distroseries.py 2011-02-22 08:20:21 +0000 | |||
16 | +++ lib/lp/registry/model/distroseries.py 2011-03-07 07:05:26 +0000 | |||
17 | @@ -1459,12 +1459,12 @@ | |||
18 | 1459 | return DecoratedResultSet(package_caches, result_to_dsbp) | 1459 | return DecoratedResultSet(package_caches, result_to_dsbp) |
19 | 1460 | 1460 | ||
20 | 1461 | def newArch(self, architecturetag, processorfamily, official, owner, | 1461 | def newArch(self, architecturetag, processorfamily, official, owner, |
22 | 1462 | supports_virtualized=False): | 1462 | supports_virtualized=False, enabled=True): |
23 | 1463 | """See `IDistroSeries`.""" | 1463 | """See `IDistroSeries`.""" |
24 | 1464 | distroarchseries = DistroArchSeries( | 1464 | distroarchseries = DistroArchSeries( |
25 | 1465 | architecturetag=architecturetag, processorfamily=processorfamily, | 1465 | architecturetag=architecturetag, processorfamily=processorfamily, |
26 | 1466 | official=official, distroseries=self, owner=owner, | 1466 | official=official, distroseries=self, owner=owner, |
28 | 1467 | supports_virtualized=supports_virtualized) | 1467 | supports_virtualized=supports_virtualized, enabled=enabled) |
29 | 1468 | return distroarchseries | 1468 | return distroarchseries |
30 | 1469 | 1469 | ||
31 | 1470 | def newMilestone(self, name, dateexpected=None, summary=None, | 1470 | def newMilestone(self, name, dateexpected=None, summary=None, |
32 | 1471 | 1471 | ||
33 | === modified file 'lib/lp/soyuz/interfaces/binarypackagerelease.py' | |||
34 | --- lib/lp/soyuz/interfaces/binarypackagerelease.py 2011-02-23 20:26:53 +0000 | |||
35 | +++ lib/lp/soyuz/interfaces/binarypackagerelease.py 2011-03-07 07:05:26 +0000 | |||
36 | @@ -44,6 +44,7 @@ | |||
37 | 44 | class IBinaryPackageRelease(Interface): | 44 | class IBinaryPackageRelease(Interface): |
38 | 45 | id = Int(title=_('ID'), required=True) | 45 | id = Int(title=_('ID'), required=True) |
39 | 46 | binarypackagename = Int(required=True) | 46 | binarypackagename = Int(required=True) |
40 | 47 | binarypackagenameID = Int(required=True) | ||
41 | 47 | version = TextLine(required=True, constraint=valid_debian_version) | 48 | version = TextLine(required=True, constraint=valid_debian_version) |
42 | 48 | summary = Text(required=True) | 49 | summary = Text(required=True) |
43 | 49 | description = Text(required=True) | 50 | description = Text(required=True) |
44 | 50 | 51 | ||
45 | === modified file 'lib/lp/soyuz/interfaces/publishing.py' | |||
46 | --- lib/lp/soyuz/interfaces/publishing.py 2011-03-03 00:43:44 +0000 | |||
47 | +++ lib/lp/soyuz/interfaces/publishing.py 2011-03-07 07:05:26 +0000 | |||
48 | @@ -678,6 +678,7 @@ | |||
49 | 678 | required=False, readonly=False, | 678 | required=False, readonly=False, |
50 | 679 | ), | 679 | ), |
51 | 680 | exported_as="distro_arch_series") | 680 | exported_as="distro_arch_series") |
52 | 681 | distroseries = Attribute("The distroseries being published into") | ||
53 | 681 | component = Int( | 682 | component = Int( |
54 | 682 | title=_('The component being published into'), | 683 | title=_('The component being published into'), |
55 | 683 | required=False, readonly=False, | 684 | required=False, readonly=False, |
56 | @@ -878,7 +879,25 @@ | |||
57 | 878 | publishing histories. | 879 | publishing histories. |
58 | 879 | """ | 880 | """ |
59 | 880 | 881 | ||
61 | 881 | def publishBinary(archive, binarypackagerelease, distroarchseries, | 882 | def publishBinaries(archive, distroseries, pocket, binaries): |
62 | 883 | """Efficiently publish multiple BinaryPackageReleases in an Archive. | ||
63 | 884 | |||
64 | 885 | Creates `IBinaryPackagePublishingHistory` records for each binary, | ||
65 | 886 | handling architecture-independent and debug packages, avoiding | ||
66 | 887 | creation of duplicate publications, and leaving disabled | ||
67 | 888 | architectures alone. | ||
68 | 889 | |||
69 | 890 | :param archive: The target `IArchive`. | ||
70 | 891 | :param distroseries: The target `IDistroSeries`. | ||
71 | 892 | :param pocket: The target `PackagePublishingPocket`. | ||
72 | 893 | :param binaries: A dict mapping `BinaryPackageReleases` to their | ||
73 | 894 | desired overrides as (`Component`, `Section`, | ||
74 | 895 | `PackagePublishingPriority`) tuples. | ||
75 | 896 | |||
76 | 897 | :return: A list of new `IBinaryPackagePublishingHistory` records. | ||
77 | 898 | """ | ||
78 | 899 | |||
79 | 900 | def publishBinary(archive, binarypackagerelease, distroseries, | ||
80 | 882 | component, section, priority, pocket): | 901 | component, section, priority, pocket): |
81 | 883 | """Publish a `BinaryPackageRelease` in an archive. | 902 | """Publish a `BinaryPackageRelease` in an archive. |
82 | 884 | 903 | ||
83 | @@ -890,9 +909,7 @@ | |||
84 | 890 | 909 | ||
85 | 891 | :param archive: The target `IArchive`. | 910 | :param archive: The target `IArchive`. |
86 | 892 | :param binarypackagerelease: The `IBinaryPackageRelease` to copy. | 911 | :param binarypackagerelease: The `IBinaryPackageRelease` to copy. |
90 | 893 | :param distroarchseries: An `IDistroArchSeries`. If the binary is | 912 | :param distroseries: An `IDistroSeries`. |
88 | 894 | architecture-independent, it will be published to all enabled | ||
89 | 895 | architectures in this series. | ||
91 | 896 | :param component: The target `IComponent`. | 913 | :param component: The target `IComponent`. |
92 | 897 | :param section: The target `ISection`. | 914 | :param section: The target `ISection`. |
93 | 898 | :param priority: The target `PackagePublishingPriority`. | 915 | :param priority: The target `PackagePublishingPriority`. |
94 | 899 | 916 | ||
95 | === modified file 'lib/lp/soyuz/model/archive.py' | |||
96 | --- lib/lp/soyuz/model/archive.py 2011-03-03 03:13:13 +0000 | |||
97 | +++ lib/lp/soyuz/model/archive.py 2011-03-07 07:05:26 +0000 | |||
98 | @@ -386,7 +386,7 @@ | |||
99 | 386 | query, clauseTables=clauseTables, orderBy=orderBy) | 386 | query, clauseTables=clauseTables, orderBy=orderBy) |
100 | 387 | return dependencies | 387 | return dependencies |
101 | 388 | 388 | ||
103 | 389 | @property | 389 | @cachedproperty |
104 | 390 | def debug_archive(self): | 390 | def debug_archive(self): |
105 | 391 | """See `IArchive`.""" | 391 | """See `IArchive`.""" |
106 | 392 | if self.purpose == ArchivePurpose.PRIMARY: | 392 | if self.purpose == ArchivePurpose.PRIMARY: |
107 | @@ -1933,6 +1933,11 @@ | |||
108 | 1933 | if enabled == False: | 1933 | if enabled == False: |
109 | 1934 | new_archive.disable() | 1934 | new_archive.disable() |
110 | 1935 | 1935 | ||
111 | 1936 | if purpose == ArchivePurpose.DEBUG: | ||
112 | 1937 | if distribution.main_archive is not None: | ||
113 | 1938 | del get_property_cache( | ||
114 | 1939 | distribution.main_archive).debug_archive | ||
115 | 1940 | |||
116 | 1936 | # Private teams cannot have public PPAs. | 1941 | # Private teams cannot have public PPAs. |
117 | 1937 | if owner.visibility == PersonVisibility.PRIVATE: | 1942 | if owner.visibility == PersonVisibility.PRIVATE: |
118 | 1938 | new_archive.buildd_secret = create_unique_token_for_table( | 1943 | new_archive.buildd_secret = create_unique_token_for_table( |
119 | 1939 | 1944 | ||
120 | === modified file 'lib/lp/soyuz/model/publishing.py' | |||
121 | --- lib/lp/soyuz/model/publishing.py 2011-03-03 08:01:34 +0000 | |||
122 | +++ lib/lp/soyuz/model/publishing.py 2011-03-07 07:05:26 +0000 | |||
123 | @@ -29,8 +29,10 @@ | |||
124 | 29 | StringCol, | 29 | StringCol, |
125 | 30 | ) | 30 | ) |
126 | 31 | from storm.expr import ( | 31 | from storm.expr import ( |
127 | 32 | And, | ||
128 | 32 | Desc, | 33 | Desc, |
129 | 33 | LeftJoin, | 34 | LeftJoin, |
130 | 35 | Or, | ||
131 | 34 | Sum, | 36 | Sum, |
132 | 35 | ) | 37 | ) |
133 | 36 | from storm.store import Store | 38 | from storm.store import Store |
134 | @@ -63,7 +65,6 @@ | |||
135 | 63 | IStoreSelector, | 65 | IStoreSelector, |
136 | 64 | MAIN_STORE, | 66 | MAIN_STORE, |
137 | 65 | ) | 67 | ) |
138 | 66 | from lp.app.errors import NotFoundError | ||
139 | 67 | from lp.buildmaster.enums import BuildStatus | 68 | from lp.buildmaster.enums import BuildStatus |
140 | 68 | from lp.buildmaster.model.buildfarmjob import BuildFarmJob | 69 | from lp.buildmaster.model.buildfarmjob import BuildFarmJob |
141 | 69 | from lp.buildmaster.model.packagebuild import PackageBuild | 70 | from lp.buildmaster.model.packagebuild import PackageBuild |
142 | @@ -121,7 +122,7 @@ | |||
143 | 121 | 'pool', poolify(source_name, component_name)) | 122 | 'pool', poolify(source_name, component_name)) |
144 | 122 | 123 | ||
145 | 123 | 124 | ||
147 | 124 | def maybe_override_component(archive, distroseries, component): | 125 | def get_component(archive, distroseries, component): |
148 | 125 | """Override the component to fit in the archive, if possible. | 126 | """Override the component to fit in the archive, if possible. |
149 | 126 | 127 | ||
150 | 127 | If the archive has a default component, and it forbids use of the | 128 | If the archive has a default component, and it forbids use of the |
151 | @@ -136,6 +137,22 @@ | |||
152 | 136 | return component | 137 | return component |
153 | 137 | 138 | ||
154 | 138 | 139 | ||
155 | 140 | def get_archive(archive, bpr): | ||
156 | 141 | """Get the archive in which this binary should be published. | ||
157 | 142 | |||
158 | 143 | Debug packages live in a DEBUG archive instead of a PRIMARY archive. | ||
159 | 144 | This helper implements that override. | ||
160 | 145 | """ | ||
161 | 146 | if bpr.binpackageformat == BinaryPackageFormat.DDEB: | ||
162 | 147 | debug_archive = archive.debug_archive | ||
163 | 148 | if debug_archive is None: | ||
164 | 149 | raise QueueInconsistentStateError( | ||
165 | 150 | "Could not find the corresponding DEBUG archive " | ||
166 | 151 | "for %s" % (archive.displayname)) | ||
167 | 152 | archive = debug_archive | ||
168 | 153 | return archive | ||
169 | 154 | |||
170 | 155 | |||
171 | 139 | class FilePublishingBase: | 156 | class FilePublishingBase: |
172 | 140 | """Base class to publish files in the archive.""" | 157 | """Base class to publish files in the archive.""" |
173 | 141 | 158 | ||
174 | @@ -914,18 +931,23 @@ | |||
175 | 914 | binarypackagepublishing=self).prejoin(preJoins) | 931 | binarypackagepublishing=self).prejoin(preJoins) |
176 | 915 | 932 | ||
177 | 916 | @property | 933 | @property |
178 | 934 | def distroseries(self): | ||
179 | 935 | """See `IBinaryPackagePublishingHistory`""" | ||
180 | 936 | return self.distroarchseries.distroseries | ||
181 | 937 | |||
182 | 938 | @property | ||
183 | 917 | def binary_package_name(self): | 939 | def binary_package_name(self): |
185 | 918 | """See `ISourcePackagePublishingHistory`""" | 940 | """See `IBinaryPackagePublishingHistory`""" |
186 | 919 | return self.binarypackagerelease.name | 941 | return self.binarypackagerelease.name |
187 | 920 | 942 | ||
188 | 921 | @property | 943 | @property |
189 | 922 | def binary_package_version(self): | 944 | def binary_package_version(self): |
191 | 923 | """See `ISourcePackagePublishingHistory`""" | 945 | """See `IBinaryPackagePublishingHistory`""" |
192 | 924 | return self.binarypackagerelease.version | 946 | return self.binarypackagerelease.version |
193 | 925 | 947 | ||
194 | 926 | @property | 948 | @property |
195 | 927 | def priority_name(self): | 949 | def priority_name(self): |
197 | 928 | """See `ISourcePackagePublishingHistory`""" | 950 | """See `IBinaryPackagePublishingHistory`""" |
198 | 929 | return self.priority.name | 951 | return self.priority.name |
199 | 930 | 952 | ||
200 | 931 | @property | 953 | @property |
201 | @@ -1245,6 +1267,42 @@ | |||
202 | 1245 | self.requestDeletion(removed_by, removal_comment) | 1267 | self.requestDeletion(removed_by, removal_comment) |
203 | 1246 | 1268 | ||
204 | 1247 | 1269 | ||
205 | 1270 | def expand_binary_requests(distroseries, binaries): | ||
206 | 1271 | """Architecture-expand a dict of binary publication requests. | ||
207 | 1272 | |||
208 | 1273 | For architecture-independent binaries, a tuple will be returned for each | ||
209 | 1274 | enabled architecture in the series. | ||
210 | 1275 | For architecture-dependent binaries, a tuple will be returned only for the | ||
211 | 1276 | architecture corresponding to the build architecture, if it exists and is | ||
212 | 1277 | enabled. | ||
213 | 1278 | |||
214 | 1279 | :param binaries: A dict mapping `BinaryPackageReleases` to tuples of their | ||
215 | 1280 | desired overrides. | ||
216 | 1281 | |||
217 | 1282 | :return: The binaries and the architectures in which they should be | ||
218 | 1283 | published, as a sequence of (`DistroArchSeries`, | ||
219 | 1284 | `BinaryPackageRelease`, (overrides)) tuples. | ||
220 | 1285 | """ | ||
221 | 1286 | |||
222 | 1287 | archs = list(distroseries.enabled_architectures) | ||
223 | 1288 | arch_map = dict((arch.architecturetag, arch) for arch in archs) | ||
224 | 1289 | |||
225 | 1290 | expanded = [] | ||
226 | 1291 | for bpr, overrides in binaries.iteritems(): | ||
227 | 1292 | if bpr.architecturespecific: | ||
228 | 1293 | # Find the DAS in this series corresponding to the original | ||
229 | 1294 | # build arch tag. If it does not exist or is disabled, we should | ||
230 | 1295 | # not publish. | ||
231 | 1296 | target_arch = arch_map.get( | ||
232 | 1297 | bpr.build.distro_arch_series.architecturetag) | ||
233 | 1298 | target_archs = [target_arch] if target_arch is not None else [] | ||
234 | 1299 | else: | ||
235 | 1300 | target_archs = archs | ||
236 | 1301 | for target_arch in target_archs: | ||
237 | 1302 | expanded.append((target_arch, bpr, overrides)) | ||
238 | 1303 | return expanded | ||
239 | 1304 | |||
240 | 1305 | |||
241 | 1248 | class PublishingSet: | 1306 | class PublishingSet: |
242 | 1249 | """Utilities for manipulating publications in batches.""" | 1307 | """Utilities for manipulating publications in batches.""" |
243 | 1250 | 1308 | ||
244 | @@ -1252,61 +1310,82 @@ | |||
245 | 1252 | 1310 | ||
246 | 1253 | def copyBinariesTo(self, binaries, distroseries, pocket, archive): | 1311 | def copyBinariesTo(self, binaries, distroseries, pocket, archive): |
247 | 1254 | """See `IPublishingSet`.""" | 1312 | """See `IPublishingSet`.""" |
270 | 1255 | secure_copies = [] | 1313 | return self.publishBinaries( |
271 | 1256 | for binary in binaries: | 1314 | archive, distroseries, pocket, |
272 | 1257 | # This will go wrong if nominatedarchindep gets deleted in a | 1315 | dict( |
273 | 1258 | # future series -- it will attempt to retrieve i386 from the | 1316 | (bpph.binarypackagerelease, (bpph.component, bpph.section, |
274 | 1259 | # new series, fail, and skip the publication instead of | 1317 | bpph.priority)) for bpph in binaries)) |
275 | 1260 | # publishing the remaining archs. | 1318 | |
276 | 1261 | try: | 1319 | def publishBinaries(self, archive, distroseries, pocket, |
277 | 1262 | build = binary.binarypackagerelease.build | 1320 | binaries): |
278 | 1263 | target_architecture = distroseries[ | 1321 | """See `IPublishingSet`.""" |
279 | 1264 | build.distro_arch_series.architecturetag] | 1322 | # Expand the dict of binaries into a list of tuples including the |
280 | 1265 | except NotFoundError: | 1323 | # architecture. |
281 | 1266 | continue | 1324 | expanded = expand_binary_requests(distroseries, binaries) |
282 | 1267 | if not target_architecture.enabled: | 1325 | |
283 | 1268 | continue | 1326 | # Find existing publications. |
284 | 1269 | secure_copies.extend( | 1327 | # We should really be able to just compare BPR.id, but |
285 | 1270 | getUtility(IPublishingSet).publishBinary( | 1328 | # CopyChecker doesn't seem to ensure that there are no |
286 | 1271 | archive, binary.binarypackagerelease, target_architecture, | 1329 | # conflicting binaries from other sources. |
287 | 1272 | binary.component, binary.section, binary.priority, | 1330 | def make_package_condition(archive, das, bpr): |
288 | 1273 | pocket)) | 1331 | return And( |
289 | 1274 | return secure_copies | 1332 | BinaryPackagePublishingHistory.archiveID == |
290 | 1275 | 1333 | get_archive(archive, bpr).id, | |
291 | 1276 | def publishBinary(self, archive, binarypackagerelease, distroarchseries, | 1334 | BinaryPackagePublishingHistory.distroarchseriesID == das.id, |
292 | 1335 | BinaryPackageRelease.binarypackagenameID == | ||
293 | 1336 | bpr.binarypackagenameID, | ||
294 | 1337 | BinaryPackageRelease.version == bpr.version, | ||
295 | 1338 | ) | ||
296 | 1339 | candidates = ( | ||
297 | 1340 | make_package_condition(archive, das, bpr) | ||
298 | 1341 | for das, bpr, overrides in expanded) | ||
299 | 1342 | already_published = IMasterStore(BinaryPackagePublishingHistory).find( | ||
300 | 1343 | (BinaryPackagePublishingHistory.distroarchseriesID, | ||
301 | 1344 | BinaryPackageRelease.binarypackagenameID, | ||
302 | 1345 | BinaryPackageRelease.version), | ||
303 | 1346 | BinaryPackagePublishingHistory.pocket == pocket, | ||
304 | 1347 | BinaryPackagePublishingHistory.status.is_in( | ||
305 | 1348 | active_publishing_status), | ||
306 | 1349 | BinaryPackageRelease.id == | ||
307 | 1350 | BinaryPackagePublishingHistory.binarypackagereleaseID, | ||
308 | 1351 | Or(*candidates)).config(distinct=True) | ||
309 | 1352 | already_published = frozenset(already_published) | ||
310 | 1353 | |||
311 | 1354 | needed = [ | ||
312 | 1355 | (das, bpr, overrides) for (das, bpr, overrides) in | ||
313 | 1356 | expanded if (das.id, bpr.binarypackagenameID, bpr.version) | ||
314 | 1357 | not in already_published] | ||
315 | 1358 | if not needed: | ||
316 | 1359 | return [] | ||
317 | 1360 | |||
318 | 1361 | insert_head = """ | ||
319 | 1362 | INSERT INTO BinaryPackagePublishingHistory | ||
320 | 1363 | (archive, distroarchseries, pocket, binarypackagerelease, | ||
321 | 1364 | component, section, priority, status, datecreated) | ||
322 | 1365 | VALUES | ||
323 | 1366 | """ | ||
324 | 1367 | insert_pubs = ", ".join( | ||
325 | 1368 | "(%s)" % ", ".join(sqlvalues( | ||
326 | 1369 | get_archive(archive, bpr).id, das.id, pocket, bpr.id, | ||
327 | 1370 | get_component(archive, das.distroseries, component).id, | ||
328 | 1371 | section.id, priority, PackagePublishingStatus.PENDING, | ||
329 | 1372 | UTC_NOW)) | ||
330 | 1373 | for (das, bpr, (component, section, priority)) in needed) | ||
331 | 1374 | insert_tail = " RETURNING BinaryPackagePublishingHistory.id" | ||
332 | 1375 | new_ids = IMasterStore(BinaryPackagePublishingHistory).execute( | ||
333 | 1376 | insert_head + insert_pubs + insert_tail) | ||
334 | 1377 | |||
335 | 1378 | publications = IMasterStore(BinaryPackagePublishingHistory).find( | ||
336 | 1379 | BinaryPackagePublishingHistory, | ||
337 | 1380 | BinaryPackagePublishingHistory.id.is_in(id[0] for id in new_ids)) | ||
338 | 1381 | return list(publications) | ||
339 | 1382 | |||
340 | 1383 | def publishBinary(self, archive, binarypackagerelease, distroseries, | ||
341 | 1277 | component, section, priority, pocket): | 1384 | component, section, priority, pocket): |
342 | 1278 | """See `IPublishingSet`.""" | 1385 | """See `IPublishingSet`.""" |
374 | 1279 | if not binarypackagerelease.architecturespecific: | 1386 | return self.publishBinaries( |
375 | 1280 | target_archs = distroarchseries.distroseries.enabled_architectures | 1387 | archive, distroseries, pocket, |
376 | 1281 | else: | 1388 | {binarypackagerelease: (component, section, priority)}) |
346 | 1282 | target_archs = [distroarchseries] | ||
347 | 1283 | |||
348 | 1284 | # DDEBs targeted to the PRIMARY archive are published in the | ||
349 | 1285 | # corresponding DEBUG archive. | ||
350 | 1286 | if binarypackagerelease.binpackageformat == BinaryPackageFormat.DDEB: | ||
351 | 1287 | debug_archive = archive.debug_archive | ||
352 | 1288 | if debug_archive is None: | ||
353 | 1289 | raise QueueInconsistentStateError( | ||
354 | 1290 | "Could not find the corresponding DEBUG archive " | ||
355 | 1291 | "for %s" % (archive.displayname)) | ||
356 | 1292 | archive = debug_archive | ||
357 | 1293 | |||
358 | 1294 | published_binaries = [] | ||
359 | 1295 | for target_arch in target_archs: | ||
360 | 1296 | # We only publish the binary if it doesn't already exist in | ||
361 | 1297 | # the destination. Note that this means we don't support | ||
362 | 1298 | # override changes on their own. | ||
363 | 1299 | binaries_in_destination = archive.getAllPublishedBinaries( | ||
364 | 1300 | name=binarypackagerelease.name, exact_match=True, | ||
365 | 1301 | version=binarypackagerelease.version, | ||
366 | 1302 | status=active_publishing_status, pocket=pocket, | ||
367 | 1303 | distroarchseries=target_arch) | ||
368 | 1304 | if not bool(binaries_in_destination): | ||
369 | 1305 | published_binaries.append( | ||
370 | 1306 | getUtility(IPublishingSet).newBinaryPublication( | ||
371 | 1307 | archive, binarypackagerelease, target_arch, component, | ||
372 | 1308 | section, priority, pocket)) | ||
373 | 1309 | return published_binaries | ||
377 | 1310 | 1389 | ||
378 | 1311 | def newBinaryPublication(self, archive, binarypackagerelease, | 1390 | def newBinaryPublication(self, archive, binarypackagerelease, |
379 | 1312 | distroarchseries, component, section, priority, | 1391 | distroarchseries, component, section, priority, |
380 | @@ -1318,7 +1397,7 @@ | |||
381 | 1318 | archive=archive, | 1397 | archive=archive, |
382 | 1319 | binarypackagerelease=binarypackagerelease, | 1398 | binarypackagerelease=binarypackagerelease, |
383 | 1320 | distroarchseries=distroarchseries, | 1399 | distroarchseries=distroarchseries, |
385 | 1321 | component=maybe_override_component( | 1400 | component=get_component( |
386 | 1322 | archive, distroarchseries.distroseries, component), | 1401 | archive, distroarchseries.distroseries, component), |
387 | 1323 | section=section, | 1402 | section=section, |
388 | 1324 | priority=priority, | 1403 | priority=priority, |
389 | @@ -1335,8 +1414,7 @@ | |||
390 | 1335 | pocket=pocket, | 1414 | pocket=pocket, |
391 | 1336 | archive=archive, | 1415 | archive=archive, |
392 | 1337 | sourcepackagerelease=sourcepackagerelease, | 1416 | sourcepackagerelease=sourcepackagerelease, |
395 | 1338 | component=maybe_override_component( | 1417 | component=get_component(archive, distroseries, component), |
394 | 1339 | archive, distroseries, component), | ||
396 | 1340 | section=section, | 1418 | section=section, |
397 | 1341 | status=PackagePublishingStatus.PENDING, | 1419 | status=PackagePublishingStatus.PENDING, |
398 | 1342 | datecreated=UTC_NOW, | 1420 | datecreated=UTC_NOW, |
399 | 1343 | 1421 | ||
400 | === modified file 'lib/lp/soyuz/model/queue.py' | |||
401 | --- lib/lp/soyuz/model/queue.py 2011-02-17 17:02:54 +0000 | |||
402 | +++ lib/lp/soyuz/model/queue.py 2011-03-07 07:05:26 +0000 | |||
403 | @@ -1457,12 +1457,9 @@ | |||
404 | 1457 | """See `IPackageUploadBuild`.""" | 1457 | """See `IPackageUploadBuild`.""" |
405 | 1458 | # Determine the build's architecturetag | 1458 | # Determine the build's architecturetag |
406 | 1459 | build_archtag = self.build.distro_arch_series.architecturetag | 1459 | build_archtag = self.build.distro_arch_series.architecturetag |
410 | 1460 | # Determine the target arch series. | 1460 | distroseries = self.packageupload.distroseries |
408 | 1461 | # This will raise NotFoundError if anything odd happens. | ||
409 | 1462 | target_das = self.packageupload.distroseries[build_archtag] | ||
411 | 1463 | debug(logger, "Publishing build to %s/%s/%s" % ( | 1461 | debug(logger, "Publishing build to %s/%s/%s" % ( |
414 | 1464 | target_das.distroseries.distribution.name, | 1462 | distroseries.distribution.name, distroseries.name, |
413 | 1465 | target_das.distroseries.name, | ||
415 | 1466 | build_archtag)) | 1463 | build_archtag)) |
416 | 1467 | 1464 | ||
417 | 1468 | # First up, publish everything in this build into that dar. | 1465 | # First up, publish everything in this build into that dar. |
418 | @@ -1478,7 +1475,7 @@ | |||
419 | 1478 | getUtility(IPublishingSet).publishBinary( | 1475 | getUtility(IPublishingSet).publishBinary( |
420 | 1479 | archive=self.packageupload.archive, | 1476 | archive=self.packageupload.archive, |
421 | 1480 | binarypackagerelease=binary, | 1477 | binarypackagerelease=binary, |
423 | 1481 | distroarchseries=target_das, | 1478 | distroseries=distroseries, |
424 | 1482 | component=binary.component, | 1479 | component=binary.component, |
425 | 1483 | section=binary.section, | 1480 | section=binary.section, |
426 | 1484 | priority=binary.priority, | 1481 | priority=binary.priority, |
427 | 1485 | 1482 | ||
428 | === modified file 'lib/lp/soyuz/tests/test_publishing.py' | |||
429 | --- lib/lp/soyuz/tests/test_publishing.py 2011-03-03 08:01:34 +0000 | |||
430 | +++ lib/lp/soyuz/tests/test_publishing.py 2011-03-07 07:05:26 +0000 | |||
431 | @@ -53,6 +53,7 @@ | |||
432 | 53 | PackagePublishingPriority, | 53 | PackagePublishingPriority, |
433 | 54 | PackagePublishingStatus, | 54 | PackagePublishingStatus, |
434 | 55 | ) | 55 | ) |
435 | 56 | from lp.soyuz.interfaces.queue import QueueInconsistentStateError | ||
436 | 56 | from lp.soyuz.interfaces.section import ISectionSet | 57 | from lp.soyuz.interfaces.section import ISectionSet |
437 | 57 | from lp.soyuz.model.processor import ProcessorFamily | 58 | from lp.soyuz.model.processor import ProcessorFamily |
438 | 58 | from lp.soyuz.model.publishing import ( | 59 | from lp.soyuz.model.publishing import ( |
439 | @@ -1399,3 +1400,113 @@ | |||
440 | 1399 | for bpf in files: | 1400 | for bpf in files: |
441 | 1400 | bpf.libraryfile.filename | 1401 | bpf.libraryfile.filename |
442 | 1401 | self.assertThat(recorder, HasQueryCount(Equals(5))) | 1402 | self.assertThat(recorder, HasQueryCount(Equals(5))) |
443 | 1403 | |||
444 | 1404 | |||
445 | 1405 | class TestPublishBinaries(TestCaseWithFactory): | ||
446 | 1406 | """Test PublishingSet.publishBinary() works.""" | ||
447 | 1407 | |||
448 | 1408 | layer = LaunchpadZopelessLayer | ||
449 | 1409 | |||
450 | 1410 | def makeArgs(self, bprs, distroseries, archive=None): | ||
451 | 1411 | """Create a dict of arguments for publishBinary.""" | ||
452 | 1412 | if archive is None: | ||
453 | 1413 | archive = distroseries.main_archive | ||
454 | 1414 | return { | ||
455 | 1415 | 'archive': archive, | ||
456 | 1416 | 'distroseries': distroseries, | ||
457 | 1417 | 'pocket': PackagePublishingPocket.BACKPORTS, | ||
458 | 1418 | 'binaries': dict( | ||
459 | 1419 | (bpr, (self.factory.makeComponent(), | ||
460 | 1420 | self.factory.makeSection(), | ||
461 | 1421 | PackagePublishingPriority.REQUIRED)) for bpr in bprs), | ||
462 | 1422 | } | ||
463 | 1423 | |||
464 | 1424 | def test_architecture_dependent(self): | ||
465 | 1425 | # Architecture-dependent binaries get created as PENDING in the | ||
466 | 1426 | # corresponding architecture of the destination series and pocket, | ||
467 | 1427 | # with the given overrides. | ||
468 | 1428 | arch_tag = self.factory.getUniqueString('arch-') | ||
469 | 1429 | orig_das = self.factory.makeDistroArchSeries( | ||
470 | 1430 | architecturetag=arch_tag) | ||
471 | 1431 | target_das = self.factory.makeDistroArchSeries( | ||
472 | 1432 | architecturetag=arch_tag) | ||
473 | 1433 | build = self.factory.makeBinaryPackageBuild(distroarchseries=orig_das) | ||
474 | 1434 | bpr = self.factory.makeBinaryPackageRelease( | ||
475 | 1435 | build=build, architecturespecific=True) | ||
476 | 1436 | args = self.makeArgs([bpr], target_das.distroseries) | ||
477 | 1437 | [bpph] = getUtility(IPublishingSet).publishBinaries(**args) | ||
478 | 1438 | overrides = args['binaries'][bpr] | ||
479 | 1439 | self.assertEqual(bpr, bpph.binarypackagerelease) | ||
480 | 1440 | self.assertEqual( | ||
481 | 1441 | (args['archive'], target_das, args['pocket']), | ||
482 | 1442 | (bpph.archive, bpph.distroarchseries, bpph.pocket)) | ||
483 | 1443 | self.assertEqual( | ||
484 | 1444 | overrides, (bpph.component, bpph.section, bpph.priority)) | ||
485 | 1445 | self.assertEqual(PackagePublishingStatus.PENDING, bpph.status) | ||
486 | 1446 | |||
487 | 1447 | def test_architecture_independent(self): | ||
488 | 1448 | # Architecture-independent binaries get published to all enabled | ||
489 | 1449 | # DASes in the series. | ||
490 | 1450 | bpr = self.factory.makeBinaryPackageRelease( | ||
491 | 1451 | architecturespecific=False) | ||
492 | 1452 | # Create 3 architectures. The binary will not be published in | ||
493 | 1453 | # the disabled one. | ||
494 | 1454 | target_das_a = self.factory.makeDistroArchSeries() | ||
495 | 1455 | target_das_b = self.factory.makeDistroArchSeries( | ||
496 | 1456 | distroseries=target_das_a.distroseries) | ||
497 | 1457 | target_das_c = self.factory.makeDistroArchSeries( | ||
498 | 1458 | distroseries=target_das_a.distroseries, enabled=False) | ||
499 | 1459 | args = self.makeArgs([bpr], target_das_a.distroseries) | ||
500 | 1460 | bpphs = getUtility(IPublishingSet).publishBinaries( | ||
501 | 1461 | **args) | ||
502 | 1462 | self.assertEquals(2, len(bpphs)) | ||
503 | 1463 | self.assertEquals( | ||
504 | 1464 | set((target_das_a, target_das_b)), | ||
505 | 1465 | set(bpph.distroarchseries for bpph in bpphs)) | ||
506 | 1466 | |||
507 | 1467 | def test_does_not_duplicate(self): | ||
508 | 1468 | # An attempt to copy something for a second time is ignored. | ||
509 | 1469 | bpr = self.factory.makeBinaryPackageRelease() | ||
510 | 1470 | target_das = self.factory.makeDistroArchSeries() | ||
511 | 1471 | args = self.makeArgs([bpr], target_das.distroseries) | ||
512 | 1472 | [new_bpph] = getUtility(IPublishingSet).publishBinaries(**args) | ||
513 | 1473 | [] = getUtility(IPublishingSet).publishBinaries(**args) | ||
514 | 1474 | |||
515 | 1475 | # But changing the target (eg. to RELEASE instead of BACKPORTS) | ||
516 | 1476 | # causes a new publication to be created. | ||
517 | 1477 | args['pocket'] = PackagePublishingPocket.RELEASE | ||
518 | 1478 | [another_bpph] = getUtility(IPublishingSet).publishBinaries(**args) | ||
519 | 1479 | |||
520 | 1480 | def test_ddebs_need_debug_archive(self): | ||
521 | 1481 | debug = self.factory.makeBinaryPackageRelease( | ||
522 | 1482 | binpackageformat=BinaryPackageFormat.DDEB) | ||
523 | 1483 | args = self.makeArgs( | ||
524 | 1484 | [debug], debug.build.distro_arch_series.distroseries) | ||
525 | 1485 | self.assertRaises( | ||
526 | 1486 | QueueInconsistentStateError, | ||
527 | 1487 | getUtility(IPublishingSet).publishBinaries, **args) | ||
528 | 1488 | |||
529 | 1489 | def test_ddebs_go_to_debug_archive(self): | ||
530 | 1490 | # Normal packages go to the given archive, but debug packages go | ||
531 | 1491 | # to the corresponding debug archive. | ||
532 | 1492 | das = self.factory.makeDistroArchSeries() | ||
533 | 1493 | self.factory.makeArchive( | ||
534 | 1494 | purpose=ArchivePurpose.DEBUG, | ||
535 | 1495 | distribution=das.distroseries.distribution) | ||
536 | 1496 | build = self.factory.makeBinaryPackageBuild(distroarchseries=das) | ||
537 | 1497 | normal = self.factory.makeBinaryPackageRelease(build=build) | ||
538 | 1498 | debug = self.factory.makeBinaryPackageRelease( | ||
539 | 1499 | build=build, binpackageformat=BinaryPackageFormat.DDEB) | ||
540 | 1500 | args = self.makeArgs([normal, debug], das.distroseries) | ||
541 | 1501 | bpphs = getUtility(IPublishingSet).publishBinaries(**args) | ||
542 | 1502 | self.assertEquals(2, len(bpphs)) | ||
543 | 1503 | self.assertEquals( | ||
544 | 1504 | set((normal, debug)), | ||
545 | 1505 | set(bpph.binarypackagerelease for bpph in bpphs)) | ||
546 | 1506 | self.assertEquals( | ||
547 | 1507 | set((das.main_archive, das.main_archive.debug_archive)), | ||
548 | 1508 | set(bpph.archive for bpph in bpphs)) | ||
549 | 1509 | |||
550 | 1510 | # A second copy does nothing, because it checks in the debug | ||
551 | 1511 | # archive too. | ||
552 | 1512 | [] = getUtility(IPublishingSet).publishBinaries(**args) | ||
553 | 1402 | 1513 | ||
554 | === modified file 'lib/lp/testing/factory.py' | |||
555 | --- lib/lp/testing/factory.py 2011-03-03 10:58:03 +0000 | |||
556 | +++ lib/lp/testing/factory.py 2011-03-07 07:05:26 +0000 | |||
557 | @@ -2315,7 +2315,7 @@ | |||
558 | 2315 | def makeDistroArchSeries(self, distroseries=None, | 2315 | def makeDistroArchSeries(self, distroseries=None, |
559 | 2316 | architecturetag=None, processorfamily=None, | 2316 | architecturetag=None, processorfamily=None, |
560 | 2317 | official=True, owner=None, | 2317 | official=True, owner=None, |
562 | 2318 | supports_virtualized=False): | 2318 | supports_virtualized=False, enabled=True): |
563 | 2319 | """Create a new distroarchseries""" | 2319 | """Create a new distroarchseries""" |
564 | 2320 | 2320 | ||
565 | 2321 | if distroseries is None: | 2321 | if distroseries is None: |
566 | @@ -2331,7 +2331,7 @@ | |||
567 | 2331 | architecturetag = self.getUniqueString('arch') | 2331 | architecturetag = self.getUniqueString('arch') |
568 | 2332 | return distroseries.newArch( | 2332 | return distroseries.newArch( |
569 | 2333 | architecturetag, processorfamily, official, owner, | 2333 | architecturetag, processorfamily, official, owner, |
571 | 2334 | supports_virtualized) | 2334 | supports_virtualized, enabled) |
572 | 2335 | 2335 | ||
573 | 2336 | def makeComponent(self, name=None): | 2336 | def makeComponent(self, name=None): |
574 | 2337 | """Make a new `IComponent`.""" | 2337 | """Make a new `IComponent`.""" |
Couple of bits that can be improved.
302 + if len(needed) == 0:
'if not needed:' would be clearer.
the 8 line generator expression is getting tricky to read.