Merge lp:~jtv/launchpad/bug-884649-branch-3 into lp:launchpad
- bug-884649-branch-3
- Merge into devel
Status: | Superseded | ||||
---|---|---|---|---|---|
Proposed branch: | lp:~jtv/launchpad/bug-884649-branch-3 | ||||
Merge into: | lp:launchpad | ||||
Diff against target: |
722 lines (+357/-129) 2 files modified
lib/lp/archivepublisher/domination.py (+202/-101) lib/lp/archivepublisher/tests/test_dominator.py (+155/-28) |
||||
To merge this branch: | bzr merge lp:~jtv/launchpad/bug-884649-branch-3 | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Launchpad code reviewers | Pending | ||
Review via email: mp+81134@code.launchpad.net |
This proposal has been superseded by a proposal from 2011-11-03.
Commit message
Limit 2nd binary-domination pass to packages with arch-indep publications.
Description of the change
= Summary =
Domination is slow. A major reason is the two-pass domination algorithm needed for binary publications.
== Proposed fix ==
The first binary-domination pass touches only architecture-
The second pass touches only architecture-
So: keep track during the first pass of which packages have architecture-
== Pre-implementation notes ==
This was Julian's idea.
== Implementation details ==
I made the second pass iterate over the intersection of “packages with multiple live publications” and “packages that were found during the first pass to have architecture-
That's not really supposed to happen, which is to say it can be helpful to be prepared for the case but it's not worth optimizing for.
== Tests ==
All the high-level desired outcomes and integration are tested in scenario tests; those remain unchanged because this is a functionally neutral optimization.
I did add one helper function, which is short but easy to mess up and so it gets its own series of tests.
{{{
./bin/test -vvc lp.archivepubli
}}}
== Demo and Q/A ==
Run the dominator. It should be tons faster, but still dominate even architecture-
= Launchpad lint =
Checking for conflicts and issues in changed files.
Linting changed files:
lib/lp/
lib/lp/
Preview Diff
1 | === modified file 'lib/lp/archivepublisher/domination.py' |
2 | --- lib/lp/archivepublisher/domination.py 2011-11-02 12:58:31 +0000 |
3 | +++ lib/lp/archivepublisher/domination.py 2011-11-03 11:52:37 +0000 |
4 | @@ -52,8 +52,12 @@ |
5 | |
6 | __all__ = ['Dominator'] |
7 | |
8 | +from collections import defaultdict |
9 | from datetime import timedelta |
10 | -from operator import itemgetter |
11 | +from operator import ( |
12 | + attrgetter, |
13 | + itemgetter, |
14 | + ) |
15 | |
16 | import apt_pkg |
17 | from storm.expr import ( |
18 | @@ -72,6 +76,7 @@ |
19 | DecoratedResultSet, |
20 | ) |
21 | from canonical.launchpad.interfaces.lpstorm import IStore |
22 | +from canonical.launchpad.utilities.orderingcheck import OrderingCheck |
23 | from lp.registry.model.sourcepackagename import SourcePackageName |
24 | from lp.services.database.bulk import load_related |
25 | from lp.soyuz.enums import ( |
26 | @@ -192,6 +197,109 @@ |
27 | else: |
28 | return version_comparison |
29 | |
30 | + def sortPublications(self, publications): |
31 | + """Sort publications from most to least current versions.""" |
32 | + # Listify; we want to iterate this twice, which won't do for a |
33 | + # non-persistent sequence. |
34 | + sorted_publications = list(publications) |
35 | + # Batch-load associated package releases; we'll be needing them |
36 | + # to compare versions. |
37 | + self.load_releases(sorted_publications) |
38 | + # Now sort. This is that second iteration. An in-place sort |
39 | + # won't hurt the original, because we're working on a copy of |
40 | + # the original iterable. |
41 | + sorted_publications.sort(cmp=self.compare, reverse=True) |
42 | + return sorted_publications |
43 | + |
44 | + |
45 | +def find_live_source_versions(publications): |
46 | + """Find versions out of Published `publications` that should stay live. |
47 | + |
48 | + This particular notion of liveness applies to source domination: the |
49 | + latest version stays live, and that's it. |
50 | + |
51 | + :param publications: An iterable of `SourcePackagePublishingHistory` |
52 | + sorted by descending package version. |
53 | + :return: A list of live versions. |
54 | + """ |
55 | + # Given the required sort order, the latest version is at the head |
56 | + # of the list. |
57 | + return [publications[0].sourcepackagerelease.version] |
58 | + |
59 | + |
60 | +def get_binary_versions(binary_publications): |
61 | + """List versions for sequence of `BinaryPackagePublishingHistory`.""" |
62 | + return [pub.binarypackagerelease.version for pub in binary_publications] |
63 | + |
64 | + |
65 | +def find_live_binary_versions_pass_1(publications): |
66 | + """Find versions out of Published `publications` that should stay live. |
67 | + |
68 | + This particular notion of liveness applies to first-pass binary |
69 | + domination: the latest version stays live, and so do publications of |
70 | + binary packages for the "all" architecture. |
71 | + |
72 | + :param publications: An iterable of `BinaryPackagePublishingHistory`, |
73 | + sorted by descending package version. |
74 | + :return: A list of live versions. |
75 | + """ |
76 | + publications = list(publications) |
77 | + latest = publications.pop(0) |
78 | + return get_binary_versions( |
79 | + [latest] + [ |
80 | + pub for pub in publications if not pub.architecture_specific]) |
81 | + |
82 | + |
83 | +def find_live_binary_versions_pass_2(publications): |
84 | + """Find versions out of Published `publications` that should stay live. |
85 | + |
86 | + This particular notion of liveness applies to second-pass binary |
87 | + domination: the latest version stays live, and architecture-specific |
88 | + publications stay live (i.e, ones that are not for the "all" |
89 | + architecture). |
90 | + |
91 | + More importantly, any publication for binary packages of the "all" |
92 | + architecture stay live if any of the non-"all" binary packages from |
93 | + the same source package release are still active -- even if they are |
94 | + for other architectures. |
95 | + |
96 | + This is the raison d'etre for the two-pass binary domination algorithm: |
97 | + to let us see which architecture-independent binary publications can be |
98 | + superseded without rendering any architecture-specific binaries from the |
99 | + same source package release uninstallable. |
100 | + |
101 | + (Note that here, "active" includes Published publications but also |
102 | + Pending ones. This is standard nomenclature in Soyuz. Some of the |
103 | + domination code confuses matters by using the term "active" to mean only |
104 | + Published publications). |
105 | + |
106 | + :param publications: An iterable of `BinaryPackagePublishingHistory`, |
107 | + sorted by descending package version. |
108 | + :return: A list of live versions. |
109 | + """ |
110 | + publications = list(publications) |
111 | + latest = publications.pop(0) |
112 | + is_arch_specific = attrgetter('architecture_specific') |
113 | + arch_specific_pubs = filter(is_arch_specific, publications) |
114 | + arch_indep_pubs = filter( |
115 | + lambda pub: not is_arch_specific(pub), |
116 | + publications) |
117 | + |
118 | + # XXX JeroenVermeulen 2011-11-01 bug=884649: This is likely to be |
119 | + # costly, and the result could be reused for all builds of the same |
120 | + # source package release to all architectures. |
121 | + reprieved_pubs = [ |
122 | + pub |
123 | + for pub in arch_indep_pubs |
124 | + if pub.getOtherPublicationsForSameSource().any()] |
125 | + |
126 | + return get_binary_versions([latest] + arch_specific_pubs + reprieved_pubs) |
127 | + |
128 | + |
129 | +def contains_arch_indep(bpphs): |
130 | + """Are any of the publications among `bpphs` architecture-independent?""" |
131 | + return any(not bpph.architecture_specific for bpph in bpphs) |
132 | + |
133 | |
134 | class Dominator: |
135 | """Manage the process of marking packages as superseded. |
136 | @@ -209,27 +317,6 @@ |
137 | self.logger = logger |
138 | self.archive = archive |
139 | |
140 | - def _checkArchIndep(self, publication): |
141 | - """Return True if the binary publication can be superseded. |
142 | - |
143 | - If the publication is an arch-indep binary, we can only supersede |
144 | - it if all the binaries from the same source are also superseded, |
145 | - else those binaries may become uninstallable. |
146 | - See bug 34086. |
147 | - """ |
148 | - binary = publication.binarypackagerelease |
149 | - if not binary.architecturespecific: |
150 | - # getOtherPublicationsForSameSource returns PENDING in |
151 | - # addition to PUBLISHED binaries, and we rely on this since |
152 | - # they must also block domination. |
153 | - others = publication.getOtherPublicationsForSameSource() |
154 | - if others.any(): |
155 | - # Don't dominate this arch:all binary as there are |
156 | - # other arch-specific binaries from the same build |
157 | - # that are still active. |
158 | - return False |
159 | - return True |
160 | - |
161 | def dominatePackage(self, publications, live_versions, generalization): |
162 | """Dominate publications for a single package. |
163 | |
164 | @@ -247,34 +334,33 @@ |
165 | |
166 | :param publications: Iterable of publications for the same package, |
167 | in the same archive, series, and pocket, all with status |
168 | - `PackagePublishingStatus.PUBLISHED`. |
169 | - :param live_versions: Iterable of version strings that are still |
170 | - considered live for this package. The given publications will |
171 | - remain active insofar as they represent any of these versions; |
172 | - older publications will be marked as superseded and newer ones |
173 | - as deleted. |
174 | + `PackagePublishingStatus.PUBLISHED`. They must be sorted from |
175 | + most current to least current, as would be the result of |
176 | + `generalization.sortPublications`. |
177 | + :param live_versions: Iterable of versions that are still considered |
178 | + "live" for this package. For any of these, the latest publication |
179 | + among `publications` will remain Published. Publications for |
180 | + older releases, as well as older publications of live versions, |
181 | + will be marked as Superseded. Publications of newer versions than |
182 | + are listed in `live_versions` are marked as Deleted. |
183 | :param generalization: A `GeneralizedPublication` helper representing |
184 | - the kind of publications these are--source or binary. |
185 | + the kind of publications these are: source or binary. |
186 | """ |
187 | - publications = list(publications) |
188 | - generalization.load_releases(publications) |
189 | - |
190 | - # Go through publications from latest version to oldest. This |
191 | - # makes it easy to figure out which release superseded which: |
192 | - # the dominant is always the oldest live release that is newer |
193 | - # than the one being superseded. In this loop, that means the |
194 | - # dominant is always the last live publication we saw. |
195 | - publications = sorted( |
196 | - publications, cmp=generalization.compare, reverse=True) |
197 | + live_versions = frozenset(live_versions) |
198 | |
199 | self.logger.debug( |
200 | "Package has %d live publication(s). Live versions: %s", |
201 | len(publications), live_versions) |
202 | |
203 | + # Verify that the publications are really sorted properly. |
204 | + check_order = OrderingCheck(cmp=generalization.compare, reverse=True) |
205 | + |
206 | current_dominant = None |
207 | dominant_version = None |
208 | |
209 | for pub in publications: |
210 | + check_order.check(pub) |
211 | + |
212 | version = generalization.getPackageVersion(pub) |
213 | # There should never be two published releases with the same |
214 | # version. So it doesn't matter whether this comparison is |
215 | @@ -295,11 +381,6 @@ |
216 | current_dominant = pub |
217 | dominant_version = version |
218 | self.logger.debug2("Keeping version %s.", version) |
219 | - elif not (generalization.is_source or self._checkArchIndep(pub)): |
220 | - # As a special case, we keep this version live as well. |
221 | - current_dominant = pub |
222 | - dominant_version = version |
223 | - self.logger.debug2("Keeping version %s.", version) |
224 | elif current_dominant is None: |
225 | # This publication is no longer live, but there is no |
226 | # newer version to supersede it either. Therefore it |
227 | @@ -312,50 +393,32 @@ |
228 | pub.supersede(current_dominant, logger=self.logger) |
229 | self.logger.debug2("Superseding version %s.", version) |
230 | |
231 | - def _dominatePublications(self, pubs, generalization): |
232 | - """Perform dominations for the given publications. |
233 | - |
234 | - Keep the latest published version for each package active, |
235 | - superseding older versions. |
236 | - |
237 | - :param pubs: A dict mapping names to a list of publications. Every |
238 | - publication must be PUBLISHED or PENDING, and the first in each |
239 | - list will be treated as dominant (so should be the latest). |
240 | - :param generalization: A `GeneralizedPublication` helper representing |
241 | - the kind of publications these are--source or binary. |
242 | - """ |
243 | - self.logger.debug("Dominating packages...") |
244 | - for name, publications in pubs.iteritems(): |
245 | - assert publications, "Empty list of publications for %s." % name |
246 | - # Since this always picks the latest version as the live |
247 | - # one, this dominatePackage call will never result in a |
248 | - # deletion. |
249 | - latest_version = generalization.getPackageVersion(publications[0]) |
250 | - self.logger.debug2("Dominating %s" % name) |
251 | - self.dominatePackage( |
252 | - publications, [latest_version], generalization) |
253 | - |
254 | - def _sortPackages(self, pkglist, generalization): |
255 | - """Map out packages by name, and sort by descending version. |
256 | - |
257 | - :param pkglist: An iterable of `SourcePackagePublishingHistory` or |
258 | - `BinaryPackagePublishingHistory`. |
259 | - :param generalization: A `GeneralizedPublication` helper representing |
260 | - the kind of publications these are--source or binary. |
261 | - :return: A dict mapping each package name to a list of publications |
262 | - from `pkglist`, newest first. |
263 | - """ |
264 | - self.logger.debug("Sorting packages...") |
265 | - |
266 | - outpkgs = {} |
267 | - for inpkg in pkglist: |
268 | - key = generalization.getPackageName(inpkg) |
269 | - outpkgs.setdefault(key, []).append(inpkg) |
270 | - |
271 | - for package_pubs in outpkgs.itervalues(): |
272 | - package_pubs.sort(cmp=generalization.compare, reverse=True) |
273 | - |
274 | - return outpkgs |
275 | + def _sortPackages(self, publications, generalization): |
276 | + """Partition publications by package name, and sort them. |
277 | + |
278 | + The publications are sorted from most current to least current, |
279 | + as required by `dominatePackage` etc. |
280 | + |
281 | + :param publications: An iterable of `SourcePackagePublishingHistory` |
282 | + or of `BinaryPackagePublishingHistory`. |
283 | + :param generalization: A `GeneralizedPublication` helper representing |
284 | + the kind of publications these are: source or binary. |
285 | + :return: A dict mapping each package name to a sorted list of |
286 | + publications from `publications`. |
287 | + """ |
288 | + pubs_by_package = defaultdict(list) |
289 | + for pub in publications: |
290 | + pubs_by_package[generalization.getPackageName(pub)].append(pub) |
291 | + |
292 | + # Sort the publication lists. This is not an in-place sort, so |
293 | + # it involves altering the dict while we iterate it. Listify |
294 | + # the keys so that we can be sure that we're not altering the |
295 | + # iteration order while iteration is underway. |
296 | + for package in list(pubs_by_package.keys()): |
297 | + pubs_by_package[package] = generalization.sortPublications( |
298 | + pubs_by_package[package]) |
299 | + |
300 | + return pubs_by_package |
301 | |
302 | def _setScheduledDeletionDate(self, pub_record): |
303 | """Set the scheduleddeletiondate on a publishing record. |
304 | @@ -510,6 +573,18 @@ |
305 | """ |
306 | generalization = GeneralizedPublication(is_source=False) |
307 | |
308 | + # Domination happens in two passes. The first tries to |
309 | + # supersede architecture-dependent publications; the second |
310 | + # tries to supersede architecture-independent ones. An |
311 | + # architecture-independent pub is kept alive as long as any |
312 | + # architecture-dependent pubs from the same source package build |
313 | + # are still live for any architecture, because they may depend |
314 | + # on the architecture-independent package. |
315 | + # Thus we limit the second pass to those packages that have |
316 | + # published, architecture-independent publications; anything |
317 | + # else will have completed domination in the first pass. |
318 | + packages_w_arch_indep = set() |
319 | + |
320 | for distroarchseries in distroseries.architectures: |
321 | self.logger.info( |
322 | "Performing domination across %s/%s (%s)", |
323 | @@ -520,21 +595,34 @@ |
324 | bins = self.findBinariesForDomination(distroarchseries, pocket) |
325 | sorted_packages = self._sortPackages(bins, generalization) |
326 | self.logger.info("Dominating binaries...") |
327 | - self._dominatePublications(sorted_packages, generalization) |
328 | - |
329 | - # We need to make a second pass to cover the cases where: |
330 | - # * arch-specific binaries were not all dominated before arch-all |
331 | - # ones that depend on them |
332 | - # * An arch-all turned into an arch-specific, or vice-versa |
333 | - # * A package is completely schizophrenic and changes all of |
334 | - # its binaries between arch-all and arch-any (apparently |
335 | - # occurs sometimes!) |
336 | + for name, pubs in sorted_packages.iteritems(): |
337 | + self.logger.debug("Dominating %s" % name) |
338 | + assert len(pubs) > 0, "Dominating zero binaries!" |
339 | + live_versions = find_live_binary_versions_pass_1(pubs) |
340 | + self.dominatePackage(pubs, live_versions, generalization) |
341 | + if contains_arch_indep(pubs): |
342 | + packages_w_arch_indep.add(name) |
343 | + |
344 | + packages_w_arch_indep = frozenset(packages_w_arch_indep) |
345 | + |
346 | + # The second pass attempts to supersede arch-all publications of |
347 | + # older versions, from source package releases that no longer |
348 | + # have any active arch-specific publications that might depend |
349 | + # on the arch-indep ones. |
350 | + # (In maintaining this code, bear in mind that some or all of a |
351 | + # source package's binary packages may switch between |
352 | + # arch-specific and arch-indep between releases.) |
353 | for distroarchseries in distroseries.architectures: |
354 | self.logger.info("Finding binaries...(2nd pass)") |
355 | bins = self.findBinariesForDomination(distroarchseries, pocket) |
356 | sorted_packages = self._sortPackages(bins, generalization) |
357 | self.logger.info("Dominating binaries...(2nd pass)") |
358 | - self._dominatePublications(sorted_packages, generalization) |
359 | + for name in packages_w_arch_indep.intersection(sorted_packages): |
360 | + pubs = sorted_packages[name] |
361 | + self.logger.debug("Dominating %s" % name) |
362 | + assert len(pubs) > 0, "Dominating zero binaries in 2nd pass!" |
363 | + live_versions = find_live_binary_versions_pass_2(pubs) |
364 | + self.dominatePackage(pubs, live_versions, generalization) |
365 | |
366 | def _composeActiveSourcePubsCondition(self, distroseries, pocket): |
367 | """Compose ORM condition for restricting relevant source pubs.""" |
368 | @@ -550,7 +638,12 @@ |
369 | ) |
370 | |
371 | def findSourcesForDomination(self, distroseries, pocket): |
372 | - """Find binary publications that need dominating.""" |
373 | + """Find binary publications that need dominating. |
374 | + |
375 | + This is only for traditional domination, where the latest published |
376 | + publication is always kept published. It will ignore publications |
377 | + that have no other publications competing for the same binary package. |
378 | + """ |
379 | # Avoid circular imports. |
380 | from lp.soyuz.model.publishing import SourcePackagePublishingHistory |
381 | |
382 | @@ -589,11 +682,18 @@ |
383 | distroseries.name, pocket.title) |
384 | |
385 | generalization = GeneralizedPublication(is_source=True) |
386 | + |
387 | + self.logger.debug("Finding sources...") |
388 | sources = self.findSourcesForDomination(distroseries, pocket) |
389 | + sorted_packages = self._sortPackages(sources, generalization) |
390 | |
391 | self.logger.debug("Dominating sources...") |
392 | - self._dominatePublications( |
393 | - self._sortPackages(sources, generalization), generalization) |
394 | + for name, pubs in sorted_packages.iteritems(): |
395 | + self.logger.debug("Dominating %s" % name) |
396 | + assert len(pubs) > 0, "Dominating zero binaries!" |
397 | + live_versions = find_live_source_versions(pubs) |
398 | + self.dominatePackage(pubs, live_versions, generalization) |
399 | + |
400 | flush_database_updates() |
401 | |
402 | def findPublishedSourcePackageNames(self, distroseries, pocket): |
403 | @@ -653,6 +753,7 @@ |
404 | """ |
405 | generalization = GeneralizedPublication(is_source=True) |
406 | pubs = self.findPublishedSPPHs(distroseries, pocket, package_name) |
407 | + pubs = generalization.sortPublications(pubs) |
408 | self.dominatePackage(pubs, live_versions, generalization) |
409 | |
410 | def judge(self, distroseries, pocket): |
411 | |
412 | === modified file 'lib/lp/archivepublisher/tests/test_dominator.py' |
413 | --- lib/lp/archivepublisher/tests/test_dominator.py 2011-11-02 10:28:31 +0000 |
414 | +++ lib/lp/archivepublisher/tests/test_dominator.py 2011-11-03 11:52:37 +0000 |
415 | @@ -15,7 +15,11 @@ |
416 | from canonical.database.sqlbase import flush_database_updates |
417 | from canonical.testing.layers import ZopelessDatabaseLayer |
418 | from lp.archivepublisher.domination import ( |
419 | + contains_arch_indep, |
420 | Dominator, |
421 | + find_live_binary_versions_pass_1, |
422 | + find_live_binary_versions_pass_2, |
423 | + find_live_source_versions, |
424 | GeneralizedPublication, |
425 | STAY_OF_EXECUTION, |
426 | ) |
427 | @@ -30,6 +34,7 @@ |
428 | StormStatementRecorder, |
429 | TestCaseWithFactory, |
430 | ) |
431 | +from lp.testing.fakemethod import FakeMethod |
432 | from lp.testing.matchers import HasQueryCount |
433 | |
434 | |
435 | @@ -72,13 +77,9 @@ |
436 | is_source=ISourcePackagePublishingHistory.providedBy(dominant)) |
437 | dominator = Dominator(self.logger, self.ubuntutest.main_archive) |
438 | |
439 | - # The _dominate* test methods require a dictionary where the |
440 | - # package name is the key. The key's value is a list of |
441 | - # source or binary packages representing dominant, the first element |
442 | - # and dominated, the subsequents. |
443 | - pubs = {'foo': [dominant, dominated]} |
444 | - |
445 | - dominator._dominatePublications(pubs, generalization) |
446 | + pubs = [dominant, dominated] |
447 | + live_versions = [generalization.getPackageVersion(dominant)] |
448 | + dominator.dominatePackage(pubs, live_versions, generalization) |
449 | flush_database_updates() |
450 | |
451 | # The dominant version remains correctly published. |
452 | @@ -158,16 +159,30 @@ |
453 | [foo_10_source] + foo_10_binaries, |
454 | PackagePublishingStatus.SUPERSEDED) |
455 | |
456 | - def testEmptyDomination(self): |
457 | - """Domination asserts for not empty input list.""" |
458 | - dominator = Dominator(self.logger, self.ubuntutest.main_archive) |
459 | - pubs = {'foo': []} |
460 | - # This isn't a really good exception. It should probably be |
461 | - # something more indicative of bad input. |
462 | - self.assertRaises( |
463 | - AssertionError, |
464 | - dominator._dominatePublications, |
465 | - pubs, GeneralizedPublication(True)) |
466 | + def test_dominateBinaries_rejects_empty_publication_list(self): |
467 | + """Domination asserts for non-empty input list.""" |
468 | + package = self.factory.makeBinaryPackageName() |
469 | + dominator = Dominator(self.logger, self.ubuntutest.main_archive) |
470 | + dominator._sortPackages = FakeMethod({package.name: []}) |
471 | + # This isn't a really good exception. It should probably be |
472 | + # something more indicative of bad input. |
473 | + self.assertRaises( |
474 | + AssertionError, |
475 | + dominator.dominateBinaries, |
476 | + self.factory.makeDistroArchSeries().distroseries, |
477 | + self.factory.getAnyPocket()) |
478 | + |
479 | + def test_dominateSources_rejects_empty_publication_list(self): |
480 | + """Domination asserts for non-empty input list.""" |
481 | + package = self.factory.makeSourcePackageName() |
482 | + dominator = Dominator(self.logger, self.ubuntutest.main_archive) |
483 | + dominator._sortPackages = FakeMethod({package.name: []}) |
484 | + # This isn't a really good exception. It should probably be |
485 | + # something more indicative of bad input. |
486 | + self.assertRaises( |
487 | + AssertionError, |
488 | + dominator.dominateSources, |
489 | + self.factory.makeDistroSeries(), self.factory.getAnyPocket()) |
490 | |
491 | def test_archall_domination(self): |
492 | # Arch-all binaries should not be dominated when a new source |
493 | @@ -358,6 +373,16 @@ |
494 | SeriesStatus.OBSOLETE) |
495 | |
496 | |
497 | +def remove_security_proxies(proxied_objects): |
498 | + """Return list of `proxied_objects`, without their proxies. |
499 | + |
500 | + The dominator runs only in scripts, where security proxies don't get |
501 | + in the way. To test realistically for this environment, strip the |
502 | + proxies wherever necessary and do as you will. |
503 | + """ |
504 | + return [removeSecurityProxy(obj) for obj in proxied_objects] |
505 | + |
506 | + |
507 | def make_spphs_for_versions(factory, versions): |
508 | """Create publication records for each of `versions`. |
509 | |
510 | @@ -400,14 +425,15 @@ |
511 | archive = das.distroseries.main_archive |
512 | pocket = factory.getAnyPocket() |
513 | bprs = [ |
514 | - factory.makeBinaryPackageRelease(binarypackagename=bpn) |
515 | + factory.makeBinaryPackageRelease( |
516 | + binarypackagename=bpn, version=version) |
517 | for version in versions] |
518 | - return [ |
519 | + return remove_security_proxies([ |
520 | factory.makeBinaryPackagePublishingHistory( |
521 | binarypackagerelease=bpr, binarypackagename=bpn, |
522 | distroarchseries=das, pocket=pocket, archive=archive, |
523 | sourcepackagename=spn, status=PackagePublishingStatus.PUBLISHED) |
524 | - for bpr in bprs] |
525 | + for bpr in bprs]) |
526 | |
527 | |
528 | def list_source_versions(spphs): |
529 | @@ -591,9 +617,10 @@ |
530 | def test_dominatePackage_supersedes_older_pub_with_newer_live_pub(self): |
531 | # When marking a package as superseded, dominatePackage |
532 | # designates a newer live version as the superseding version. |
533 | + generalization = GeneralizedPublication(True) |
534 | pubs = make_spphs_for_versions(self.factory, ['1.0', '1.1']) |
535 | self.makeDominator(pubs).dominatePackage( |
536 | - pubs, ['1.1'], GeneralizedPublication(True)) |
537 | + generalization.sortPublications(pubs), ['1.1'], generalization) |
538 | self.assertEqual(PackagePublishingStatus.SUPERSEDED, pubs[0].status) |
539 | self.assertEqual(pubs[1].sourcepackagerelease, pubs[0].supersededby) |
540 | self.assertEqual(PackagePublishingStatus.PUBLISHED, pubs[1].status) |
541 | @@ -601,10 +628,11 @@ |
542 | def test_dominatePackage_only_supersedes_with_live_pub(self): |
543 | # When marking a package as superseded, dominatePackage will |
544 | # only pick a live version as the superseding one. |
545 | + generalization = GeneralizedPublication(True) |
546 | pubs = make_spphs_for_versions( |
547 | self.factory, ['1.0', '2.0', '3.0', '4.0']) |
548 | self.makeDominator(pubs).dominatePackage( |
549 | - pubs, ['3.0'], GeneralizedPublication(True)) |
550 | + generalization.sortPublications(pubs), ['3.0'], generalization) |
551 | self.assertEqual([ |
552 | pubs[2].sourcepackagerelease, |
553 | pubs[2].sourcepackagerelease, |
554 | @@ -616,23 +644,27 @@ |
555 | def test_dominatePackage_supersedes_with_oldest_newer_live_pub(self): |
556 | # When marking a package as superseded, dominatePackage picks |
557 | # the oldest of the newer, live versions as the superseding one. |
558 | + generalization = GeneralizedPublication(True) |
559 | pubs = make_spphs_for_versions(self.factory, ['2.7', '2.8', '2.9']) |
560 | self.makeDominator(pubs).dominatePackage( |
561 | - pubs, ['2.8', '2.9'], GeneralizedPublication(True)) |
562 | + generalization.sortPublications(pubs), ['2.8', '2.9'], |
563 | + generalization) |
564 | self.assertEqual(pubs[1].sourcepackagerelease, pubs[0].supersededby) |
565 | |
566 | def test_dominatePackage_only_supersedes_with_newer_live_pub(self): |
567 | # When marking a package as superseded, dominatePackage only |
568 | # considers a newer version as the superseding one. |
569 | + generalization = GeneralizedPublication(True) |
570 | pubs = make_spphs_for_versions(self.factory, ['0.1', '0.2']) |
571 | self.makeDominator(pubs).dominatePackage( |
572 | - pubs, ['0.1'], GeneralizedPublication(True)) |
573 | + generalization.sortPublications(pubs), ['0.1'], generalization) |
574 | self.assertEqual(None, pubs[1].supersededby) |
575 | self.assertEqual(PackagePublishingStatus.DELETED, pubs[1].status) |
576 | |
577 | def test_dominatePackage_supersedes_replaced_pub_for_live_version(self): |
578 | # Even if a publication record is for a live version, a newer |
579 | # one for the same version supersedes it. |
580 | + generalization = GeneralizedPublication(True) |
581 | spr = self.factory.makeSourcePackageRelease() |
582 | series = self.factory.makeDistroSeries() |
583 | pocket = PackagePublishingPocket.RELEASE |
584 | @@ -649,7 +681,8 @@ |
585 | ]) |
586 | |
587 | self.makeDominator(pubs).dominatePackage( |
588 | - pubs, [spr.version], GeneralizedPublication(True)) |
589 | + generalization.sortPublications(pubs), [spr.version], |
590 | + generalization) |
591 | self.assertEqual([ |
592 | PackagePublishingStatus.SUPERSEDED, |
593 | PackagePublishingStatus.SUPERSEDED, |
594 | @@ -661,12 +694,13 @@ |
595 | |
596 | def test_dominatePackage_is_efficient(self): |
597 | # dominatePackage avoids issuing too many queries. |
598 | + generalization = GeneralizedPublication(True) |
599 | versions = ["1.%s" % revision for revision in xrange(5)] |
600 | pubs = make_spphs_for_versions(self.factory, versions) |
601 | with StormStatementRecorder() as recorder: |
602 | self.makeDominator(pubs).dominatePackage( |
603 | - pubs, versions[2:-1], |
604 | - GeneralizedPublication(True)) |
605 | + generalization.sortPublications(pubs), versions[2:-1], |
606 | + generalization) |
607 | self.assertThat(recorder, HasQueryCount(LessThan(5))) |
608 | |
609 | def test_dominatePackage_advanced_scenario(self): |
610 | @@ -677,6 +711,7 @@ |
611 | # don't just patch up the code or this test. Create unit tests |
612 | # that specifically cover the difference, then change the code |
613 | # and/or adapt this test to return to harmony. |
614 | + generalization = GeneralizedPublication(True) |
615 | series = self.factory.makeDistroSeries() |
616 | package = self.factory.makeSourcePackageName() |
617 | pocket = PackagePublishingPocket.RELEASE |
618 | @@ -723,7 +758,8 @@ |
619 | |
620 | all_pubs = sum(pubs_by_version.itervalues(), []) |
621 | Dominator(DevNullLogger(), series.main_archive).dominatePackage( |
622 | - all_pubs, live_versions, GeneralizedPublication(True)) |
623 | + generalization.sortPublications(all_pubs), live_versions, |
624 | + generalization) |
625 | |
626 | for version in reversed(versions): |
627 | pubs = pubs_by_version[version] |
628 | @@ -1089,3 +1125,94 @@ |
629 | published_spphs, |
630 | dominator.findSourcesForDomination( |
631 | spphs[0].distroseries, spphs[0].pocket)) |
632 | + |
633 | + |
634 | +def make_publications_arch_specific(pubs, arch_specific=True): |
635 | + """Set the `architecturespecific` attribute for given SPPHs. |
636 | + |
637 | + :param pubs: An iterable of `BinaryPackagePublishingHistory`. |
638 | + :param arch_specific: Whether the binary package releases published |
639 | + by `pubs` are to be architecture-specific. If not, they will be |
640 | + treated as being for the "all" architecture. |
641 | + """ |
642 | + for pub in pubs: |
643 | + bpr = removeSecurityProxy(pub).binarypackagerelease |
644 | + bpr.architecturespecific = arch_specific |
645 | + |
646 | + |
647 | +class TestLivenessFunctions(TestCaseWithFactory): |
648 | + """Tests for the functions that say which versions are live.""" |
649 | + |
650 | + layer = ZopelessDatabaseLayer |
651 | + |
652 | + def test_find_live_source_versions_blesses_latest(self): |
653 | + spphs = make_spphs_for_versions(self.factory, ['1.2', '1.1', '1.0']) |
654 | + self.assertEqual(['1.2'], find_live_source_versions(spphs)) |
655 | + |
656 | + def test_find_live_binary_versions_pass_1_blesses_latest(self): |
657 | + bpphs = make_bpphs_for_versions(self.factory, ['1.2', '1.1', '1.0']) |
658 | + make_publications_arch_specific(bpphs) |
659 | + self.assertEqual(['1.2'], find_live_binary_versions_pass_1(bpphs)) |
660 | + |
661 | + def test_find_live_binary_versions_pass_1_blesses_arch_all(self): |
662 | + versions = list(reversed(['1.%d' % version for version in range(3)])) |
663 | + bpphs = make_bpphs_for_versions(self.factory, versions) |
664 | + |
665 | + # All of these publications are architecture-specific, except |
666 | + # the last one. This would happen if the binary package had |
667 | + # just changed from being architecture-specific to being |
668 | + # architecture-independent. |
669 | + make_publications_arch_specific(bpphs, True) |
670 | + make_publications_arch_specific(bpphs[-1:], False) |
671 | + self.assertEqual( |
672 | + versions[:1] + versions[-1:], |
673 | + find_live_binary_versions_pass_1(bpphs)) |
674 | + |
675 | + def test_find_live_binary_versions_pass_2_blesses_latest(self): |
676 | + bpphs = make_bpphs_for_versions(self.factory, ['1.2', '1.1', '1.0']) |
677 | + make_publications_arch_specific(bpphs, False) |
678 | + self.assertEqual(['1.2'], find_live_binary_versions_pass_2(bpphs)) |
679 | + |
680 | + def test_find_live_binary_versions_pass_2_blesses_arch_specific(self): |
681 | + versions = list(reversed(['1.%d' % version for version in range(3)])) |
682 | + bpphs = make_bpphs_for_versions(self.factory, versions) |
683 | + make_publications_arch_specific(bpphs) |
684 | + self.assertEqual(versions, find_live_binary_versions_pass_2(bpphs)) |
685 | + |
686 | + def test_find_live_binary_versions_pass_2_reprieves_arch_all(self): |
687 | + # An arch-all BPPH for a BPR built by an SPR that also still has |
688 | + # active arch-dependent BPPHs gets a reprieve: it can't be |
689 | + # superseded until those arch-dependent BPPHs have been |
690 | + # superseded. |
691 | + bpphs = make_bpphs_for_versions(self.factory, ['1.2', '1.1', '1.0']) |
692 | + make_publications_arch_specific(bpphs, False) |
693 | + dependent = self.factory.makeBinaryPackagePublishingHistory( |
694 | + binarypackagerelease=bpphs[1].binarypackagerelease) |
695 | + make_publications_arch_specific([dependent], True) |
696 | + self.assertEqual( |
697 | + ['1.2', '1.1'], find_live_binary_versions_pass_2(bpphs)) |
698 | + |
699 | + |
700 | +class TestDominationHelpers(TestCaseWithFactory): |
701 | + """Test lightweight helpers for the `Dominator`.""" |
702 | + |
703 | + layer = ZopelessDatabaseLayer |
704 | + |
705 | + def test_contains_arch_indep_says_True_for_arch_indep(self): |
706 | + bpphs = [self.factory.makeBinaryPackagePublishingHistory()] |
707 | + make_publications_arch_specific(bpphs, False) |
708 | + self.assertTrue(contains_arch_indep(bpphs)) |
709 | + |
710 | + def test_contains_arch_indep_says_False_for_arch_specific(self): |
711 | + bpphs = [self.factory.makeBinaryPackagePublishingHistory()] |
712 | + make_publications_arch_specific(bpphs, True) |
713 | + self.assertFalse(contains_arch_indep(bpphs)) |
714 | + |
715 | + def test_contains_arch_indep_says_True_for_combination(self): |
716 | + bpphs = make_bpphs_for_versions(self.factory, ['1.1', '1.0']) |
717 | + make_publications_arch_specific(bpphs[:1], True) |
718 | + make_publications_arch_specific(bpphs[1:], False) |
719 | + self.assertTrue(contains_arch_indep(bpphs)) |
720 | + |
721 | + def test_contains_arch_indep_says_False_for_empty_list(self): |
722 | + self.assertFalse(contains_arch_indep([])) |