Merge ~cjwatson/launchpad:dominator-channel-map into launchpad:master
- Git
- lp:~cjwatson/launchpad
- dominator-channel-map
- Merge into master
Proposed by
Colin Watson
Status: | Merged |
---|---|
Approved by: | Colin Watson |
Approved revision: | 038e321722bbc735367ae64ce92dac32aa528f80 |
Merge reported by: | Otto Co-Pilot |
Merged at revision: | not available |
Proposed branch: | ~cjwatson/launchpad:dominator-channel-map |
Merge into: | launchpad:master |
Diff against target: |
1347 lines (+456/-124) 14 files modified
lib/lp/archivepublisher/domination.py (+73/-26) lib/lp/archivepublisher/publishing.py (+3/-0) lib/lp/archivepublisher/tests/test_dominator.py (+74/-9) lib/lp/code/interfaces/cibuild.py (+8/-0) lib/lp/code/model/cibuild.py (+13/-0) lib/lp/code/model/tests/test_cibuild.py (+19/-0) lib/lp/soyuz/adapters/packagelocation.py (+27/-5) lib/lp/soyuz/adapters/tests/test_packagelocation.py (+20/-2) lib/lp/soyuz/enums.py (+14/-0) lib/lp/soyuz/interfaces/publishing.py (+6/-8) lib/lp/soyuz/model/binarypackagerelease.py (+2/-0) lib/lp/soyuz/model/publishing.py (+78/-33) lib/lp/soyuz/tests/test_publishing.py (+70/-20) lib/lp/testing/factory.py (+49/-21) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant | code | Approve | |
Review via email: mp+422233@code.launchpad.net |
Commit message
Support dominating publications by channel
Description of the change
This means that, once we start publishing binaries to snap-store-style semantic channels (e.g. "stable", "1.0/candidate", etc.), the dominator will be able to correctly supersede publications that have been replaced with a different version of the same package in the same channel.
I had to prepare for this by adding channel support to various publishing primitives. There's no UI or API support for any of this yet, but I at least needed enough to support the dominator and its tests.
To post a comment you must log in.
Revision history for this message
William Grant (wgrant) : | # |
review:
Approve
(code)
Revision history for this message
Colin Watson (cjwatson) : | # |
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/lib/lp/archivepublisher/domination.py b/lib/lp/archivepublisher/domination.py |
2 | index 5cc9422..526c8ea 100644 |
3 | --- a/lib/lp/archivepublisher/domination.py |
4 | +++ b/lib/lp/archivepublisher/domination.py |
5 | @@ -64,6 +64,7 @@ from storm.expr import ( |
6 | And, |
7 | Count, |
8 | Desc, |
9 | + Not, |
10 | Select, |
11 | ) |
12 | from zope.component import getUtility |
13 | @@ -74,7 +75,9 @@ from lp.services.database.constants import UTC_NOW |
14 | from lp.services.database.decoratedresultset import DecoratedResultSet |
15 | from lp.services.database.interfaces import IStore |
16 | from lp.services.database.sqlbase import flush_database_updates |
17 | +from lp.services.database.stormexpr import IsDistinctFrom |
18 | from lp.services.orderingcheck import OrderingCheck |
19 | +from lp.soyuz.adapters.packagelocation import PackageLocation |
20 | from lp.soyuz.enums import ( |
21 | BinaryPackageFormat, |
22 | PackagePublishingStatus, |
23 | @@ -204,6 +207,17 @@ class GeneralizedPublication: |
24 | return sorted(publications, key=cmp_to_key(self.compare), reverse=True) |
25 | |
26 | |
27 | +def make_package_location(pub): |
28 | + """Make a `PackageLocation` representing a publication.""" |
29 | + return PackageLocation( |
30 | + archive=pub.archive, |
31 | + distribution=pub.distroseries.distribution, |
32 | + distroseries=pub.distroseries, |
33 | + pocket=pub.pocket, |
34 | + channel=pub.channel, |
35 | + ) |
36 | + |
37 | + |
38 | def find_live_source_versions(sorted_pubs): |
39 | """Find versions out of Published publications that should stay live. |
40 | |
41 | @@ -336,10 +350,19 @@ def find_live_binary_versions_pass_2(sorted_pubs, cache): |
42 | [pub.binarypackagerelease for pub in arch_indep_pubs], ['buildID']) |
43 | load_related(SourcePackageRelease, bpbs, ['source_package_release_id']) |
44 | |
45 | + # XXX cjwatson 2022-05-01: Skip the architecture-specific check for |
46 | + # publications from CI builds for now, until we figure out how to |
47 | + # approximate source package releases for groups of CI builds. We don't |
48 | + # currently expect problematic situations to come up on production; CI |
49 | + # builds are currently only expected to be used in situations where |
50 | + # either we don't build both architecture-specific and |
51 | + # architecture-independent packages, or where tight dependencies between |
52 | + # the two aren't customary. |
53 | reprieved_pubs = [ |
54 | pub |
55 | for pub in arch_indep_pubs |
56 | - if cache.hasArchSpecificPublications(pub)] |
57 | + if pub.binarypackagerelease.ci_build_id is None and |
58 | + cache.hasArchSpecificPublications(pub)] |
59 | |
60 | return get_binary_versions([latest] + arch_specific_pubs + reprieved_pubs) |
61 | |
62 | @@ -382,9 +405,9 @@ class Dominator: |
63 | list we import. |
64 | |
65 | :param sorted_pubs: A list of publications for the same package, |
66 | - in the same archive, series, and pocket, all with status |
67 | - `PackagePublishingStatus.PUBLISHED`. They must be sorted from |
68 | - most current to least current, as would be the result of |
69 | + in the same archive, series, pocket, and channel, all with |
70 | + status `PackagePublishingStatus.PUBLISHED`. They must be sorted |
71 | + from most current to least current, as would be the result of |
72 | `generalization.sortPublications`. |
73 | :param live_versions: Iterable of versions that are still considered |
74 | "live" for this package. For any of these, the latest publication |
75 | @@ -458,31 +481,47 @@ class Dominator: |
76 | return supersede, keep, delete |
77 | |
78 | def _sortPackages(self, publications, generalization): |
79 | - """Partition publications by package name, and sort them. |
80 | + """Partition publications by package name and location, and sort them. |
81 | |
82 | The publications are sorted from most current to least current, |
83 | - as required by `planPackageDomination` etc. |
84 | + as required by `planPackageDomination` etc. Locations are currently |
85 | + (package name, channel). |
86 | |
87 | :param publications: An iterable of `SourcePackagePublishingHistory` |
88 | or of `BinaryPackagePublishingHistory`. |
89 | :param generalization: A `GeneralizedPublication` helper representing |
90 | the kind of publications these are: source or binary. |
91 | - :return: A dict mapping each package name to a sorted list of |
92 | - publications from `publications`. |
93 | + :return: A dict mapping each package location (package name, |
94 | + channel) to a sorted list of publications from `publications`. |
95 | """ |
96 | - pubs_by_package = defaultdict(list) |
97 | + # XXX cjwatson 2022-05-19: Traditional suites (distroseries/pocket) |
98 | + # are divided up in the loop in Publisher.B_dominate. However, this |
99 | + # doesn't scale to channel-map-style suites (distroseries/channel), |
100 | + # since there may be a very large number of channels and we don't |
101 | + # want to have to loop over all the possible ones, so we divide |
102 | + # those up here instead. |
103 | + # |
104 | + # This is definitely confusing. In the longer term, we should |
105 | + # probably push the loop down from the publisher to here, and sort |
106 | + # and dominate all candidates in a given archive at once: there's no |
107 | + # particularly obvious reason not to, and it might perform better as |
108 | + # well. |
109 | + |
110 | + pubs_by_name_and_location = defaultdict(list) |
111 | for pub in publications: |
112 | - pubs_by_package[generalization.getPackageName(pub)].append(pub) |
113 | + name = generalization.getPackageName(pub) |
114 | + location = make_package_location(pub) |
115 | + pubs_by_name_and_location[(name, location)].append(pub) |
116 | |
117 | # Sort the publication lists. This is not an in-place sort, so |
118 | # it involves altering the dict while we iterate it. Listify |
119 | - # the keys so that we can be sure that we're not altering the |
120 | + # the items so that we can be sure that we're not altering the |
121 | # iteration order while iteration is underway. |
122 | - for package in list(pubs_by_package.keys()): |
123 | - pubs_by_package[package] = generalization.sortPublications( |
124 | - pubs_by_package[package]) |
125 | + for (name, location), pubs in list(pubs_by_name_and_location.items()): |
126 | + pubs_by_name_and_location[(name, location)] = ( |
127 | + generalization.sortPublications(pubs)) |
128 | |
129 | - return pubs_by_package |
130 | + return pubs_by_name_and_location |
131 | |
132 | def _setScheduledDeletionDate(self, pub_record): |
133 | """Set the scheduleddeletiondate on a publishing record. |
134 | @@ -541,7 +580,10 @@ class Dominator: |
135 | BinaryPackagePublishingHistory.binarypackagerelease == |
136 | BinaryPackageRelease.id, |
137 | BinaryPackageRelease.build == BinaryPackageBuild.id, |
138 | - BinaryPackagePublishingHistory.pocket == pub_record.pocket) |
139 | + BinaryPackagePublishingHistory.pocket == pub_record.pocket, |
140 | + Not(IsDistinctFrom( |
141 | + BinaryPackagePublishingHistory._channel, |
142 | + pub_record._channel))) |
143 | |
144 | # There is at least one non-removed binary to consider |
145 | if not considered_binaries.is_empty(): |
146 | @@ -552,6 +594,7 @@ class Dominator: |
147 | SourcePackagePublishingHistory, |
148 | distroseries=pub_record.distroseries, |
149 | pocket=pub_record.pocket, |
150 | + channel=pub_record.channel, |
151 | status=PackagePublishingStatus.PUBLISHED, |
152 | archive=self.archive, |
153 | sourcepackagerelease=srcpkg_release) |
154 | @@ -588,7 +631,8 @@ class Dominator: |
155 | ] |
156 | candidate_binary_names = Select( |
157 | BPPH.binarypackagenameID, And(*bpph_location_clauses), |
158 | - group_by=BPPH.binarypackagenameID, having=(Count() > 1)) |
159 | + group_by=(BPPH.binarypackagenameID, BPPH._channel), |
160 | + having=(Count() > 1)) |
161 | main_clauses = bpph_location_clauses + [ |
162 | BPR.id == BPPH.binarypackagereleaseID, |
163 | BPR.binarypackagenameID.is_in(candidate_binary_names), |
164 | @@ -664,13 +708,14 @@ class Dominator: |
165 | bins = self.findBinariesForDomination(distroarchseries, pocket) |
166 | sorted_packages = self._sortPackages(bins, generalization) |
167 | self.logger.info("Planning domination of binaries...") |
168 | - for name, pubs in sorted_packages.items(): |
169 | - self.logger.debug("Planning domination of %s" % name) |
170 | + for (name, location), pubs in sorted_packages.items(): |
171 | + self.logger.debug( |
172 | + "Planning domination of %s in %s" % (name, location)) |
173 | assert len(pubs) > 0, "Dominating zero binaries!" |
174 | live_versions = find_live_binary_versions_pass_1(pubs) |
175 | plan(pubs, live_versions) |
176 | if contains_arch_indep(pubs): |
177 | - packages_w_arch_indep.add(name) |
178 | + packages_w_arch_indep.add((name, location)) |
179 | |
180 | execute_plan() |
181 | |
182 | @@ -692,9 +737,11 @@ class Dominator: |
183 | bins = self.findBinariesForDomination(distroarchseries, pocket) |
184 | sorted_packages = self._sortPackages(bins, generalization) |
185 | self.logger.info("Planning domination of binaries...(2nd pass)") |
186 | - for name in packages_w_arch_indep.intersection(sorted_packages): |
187 | - pubs = sorted_packages[name] |
188 | - self.logger.debug("Planning domination of %s" % name) |
189 | + for name, location in packages_w_arch_indep.intersection( |
190 | + sorted_packages): |
191 | + pubs = sorted_packages[(name, location)] |
192 | + self.logger.debug( |
193 | + "Planning domination of %s in %s" % (name, location)) |
194 | assert len(pubs) > 0, "Dominating zero binaries in 2nd pass!" |
195 | live_versions = find_live_binary_versions_pass_2( |
196 | pubs, reprieve_cache) |
197 | @@ -732,7 +779,7 @@ class Dominator: |
198 | candidate_source_names = Select( |
199 | SPPH.sourcepackagenameID, |
200 | And(join_spph_spr(), spph_location_clauses), |
201 | - group_by=SPPH.sourcepackagenameID, |
202 | + group_by=(SPPH.sourcepackagenameID, SPPH._channel), |
203 | having=(Count() > 1)) |
204 | |
205 | # We'll also access the SourcePackageReleases associated with |
206 | @@ -769,8 +816,8 @@ class Dominator: |
207 | delete = [] |
208 | |
209 | self.logger.debug("Dominating sources...") |
210 | - for name, pubs in sorted_packages.items(): |
211 | - self.logger.debug("Dominating %s" % name) |
212 | + for (name, location), pubs in sorted_packages.items(): |
213 | + self.logger.debug("Dominating %s in %s" % (name, location)) |
214 | assert len(pubs) > 0, "Dominating zero sources!" |
215 | live_versions = find_live_source_versions(pubs) |
216 | cur_supersede, _, cur_delete = self.planPackageDomination( |
217 | diff --git a/lib/lp/archivepublisher/publishing.py b/lib/lp/archivepublisher/publishing.py |
218 | index 0639716..69fb7e3 100644 |
219 | --- a/lib/lp/archivepublisher/publishing.py |
220 | +++ b/lib/lp/archivepublisher/publishing.py |
221 | @@ -680,6 +680,9 @@ class Publisher: |
222 | judgejudy = Dominator(self.log, self.archive) |
223 | for distroseries in self.distro.series: |
224 | for pocket in self.archive.getPockets(): |
225 | + # XXX cjwatson 2022-05-19: Channels are handled in the |
226 | + # dominator instead; see the comment in |
227 | + # Dominator._sortPackages. |
228 | if not self.isAllowed(distroseries, pocket): |
229 | continue |
230 | if not force_domination: |
231 | diff --git a/lib/lp/archivepublisher/tests/test_dominator.py b/lib/lp/archivepublisher/tests/test_dominator.py |
232 | old mode 100755 |
233 | new mode 100644 |
234 | index 01c56e4..23d5468 |
235 | --- a/lib/lp/archivepublisher/tests/test_dominator.py |
236 | +++ b/lib/lp/archivepublisher/tests/test_dominator.py |
237 | @@ -30,7 +30,11 @@ from lp.archivepublisher.publishing import Publisher |
238 | from lp.registry.interfaces.pocket import PackagePublishingPocket |
239 | from lp.registry.interfaces.series import SeriesStatus |
240 | from lp.services.log.logger import DevNullLogger |
241 | -from lp.soyuz.enums import PackagePublishingStatus |
242 | +from lp.soyuz.adapters.packagelocation import PackageLocation |
243 | +from lp.soyuz.enums import ( |
244 | + BinaryPackageFormat, |
245 | + PackagePublishingStatus, |
246 | + ) |
247 | from lp.soyuz.interfaces.publishing import ( |
248 | IPublishingSet, |
249 | ISourcePackagePublishingHistory, |
250 | @@ -168,29 +172,39 @@ class TestDominator(TestNativePublishingBase): |
251 | """Domination asserts for non-empty input list.""" |
252 | with lp_dbuser(): |
253 | distroseries = self.factory.makeDistroArchSeries().distroseries |
254 | + pocket = self.factory.getAnyPocket() |
255 | package = self.factory.makeBinaryPackageName() |
256 | + location = PackageLocation( |
257 | + archive=self.ubuntutest.main_archive, |
258 | + distribution=distroseries.distribution, distroseries=distroseries, |
259 | + pocket=pocket) |
260 | dominator = Dominator(self.logger, self.ubuntutest.main_archive) |
261 | - dominator._sortPackages = FakeMethod({package.name: []}) |
262 | + dominator._sortPackages = FakeMethod({(package.name, location): []}) |
263 | # This isn't a really good exception. It should probably be |
264 | # something more indicative of bad input. |
265 | self.assertRaises( |
266 | AssertionError, |
267 | dominator.dominateBinaries, |
268 | - distroseries, self.factory.getAnyPocket()) |
269 | + distroseries, pocket) |
270 | |
271 | def test_dominateSources_rejects_empty_publication_list(self): |
272 | """Domination asserts for non-empty input list.""" |
273 | with lp_dbuser(): |
274 | distroseries = self.factory.makeDistroSeries() |
275 | + pocket = self.factory.getAnyPocket() |
276 | package = self.factory.makeSourcePackageName() |
277 | + location = PackageLocation( |
278 | + archive=self.ubuntutest.main_archive, |
279 | + distribution=distroseries.distribution, distroseries=distroseries, |
280 | + pocket=pocket) |
281 | dominator = Dominator(self.logger, self.ubuntutest.main_archive) |
282 | - dominator._sortPackages = FakeMethod({package.name: []}) |
283 | + dominator._sortPackages = FakeMethod({(package.name, location): []}) |
284 | # This isn't a really good exception. It should probably be |
285 | # something more indicative of bad input. |
286 | self.assertRaises( |
287 | AssertionError, |
288 | dominator.dominateSources, |
289 | - distroseries, self.factory.getAnyPocket()) |
290 | + distroseries, pocket) |
291 | |
292 | def test_archall_domination(self): |
293 | # Arch-all binaries should not be dominated when a new source |
294 | @@ -398,6 +412,51 @@ class TestDominator(TestNativePublishingBase): |
295 | for pub in overrides_2: |
296 | self.assertEqual(PackagePublishingStatus.PUBLISHED, pub.status) |
297 | |
298 | + def test_dominate_by_channel(self): |
299 | + # Publications only dominate other publications in the same channel. |
300 | + # (Currently only tested for binary publications.) |
301 | + with lp_dbuser(): |
302 | + archive = self.factory.makeArchive() |
303 | + distroseries = self.factory.makeDistroSeries( |
304 | + distribution=archive.distribution) |
305 | + das = self.factory.makeDistroArchSeries(distroseries=distroseries) |
306 | + repository = self.factory.makeGitRepository( |
307 | + target=self.factory.makeDistributionSourcePackage( |
308 | + distribution=archive.distribution)) |
309 | + ci_builds = [ |
310 | + self.factory.makeCIBuild( |
311 | + git_repository=repository, distro_arch_series=das) |
312 | + for _ in range(3)] |
313 | + bpn = self.factory.makeBinaryPackageName() |
314 | + bprs = [ |
315 | + self.factory.makeBinaryPackageRelease( |
316 | + binarypackagename=bpn, version=version, ci_build=ci_build, |
317 | + binpackageformat=BinaryPackageFormat.WHL) |
318 | + for version, ci_build in zip(("1.0", "1.1", "1.2"), ci_builds)] |
319 | + stable_bpphs = [ |
320 | + self.factory.makeBinaryPackagePublishingHistory( |
321 | + binarypackagerelease=bpr, archive=archive, |
322 | + distroarchseries=das, status=PackagePublishingStatus.PUBLISHED, |
323 | + pocket=PackagePublishingPocket.RELEASE, channel="stable") |
324 | + for bpr in bprs[:2]] |
325 | + candidate_bpph = self.factory.makeBinaryPackagePublishingHistory( |
326 | + binarypackagerelease=bprs[2], archive=archive, |
327 | + distroarchseries=das, status=PackagePublishingStatus.PUBLISHED, |
328 | + pocket=PackagePublishingPocket.RELEASE, channel="candidate") |
329 | + |
330 | + dominator = Dominator(self.logger, archive) |
331 | + dominator.judgeAndDominate( |
332 | + distroseries, PackagePublishingPocket.RELEASE) |
333 | + |
334 | + # The older of the two stable publications is superseded, while the |
335 | + # current stable publication and the candidate publication are left |
336 | + # alone. |
337 | + self.checkPublication( |
338 | + stable_bpphs[0], PackagePublishingStatus.SUPERSEDED) |
339 | + self.checkPublications( |
340 | + (stable_bpphs[1], candidate_bpph), |
341 | + PackagePublishingStatus.PUBLISHED) |
342 | + |
343 | |
344 | class TestDomination(TestNativePublishingBase): |
345 | """Test overall domination procedure.""" |
346 | @@ -1315,7 +1374,7 @@ class TestArchSpecificPublicationsCache(TestCaseWithFactory): |
347 | return removeSecurityProxy(self.factory.makeSourcePackageRelease()) |
348 | |
349 | def makeBPPH(self, spr=None, arch_specific=True, archive=None, |
350 | - distroseries=None): |
351 | + distroseries=None, binpackageformat=None, channel=None): |
352 | """Create a `BinaryPackagePublishingHistory`.""" |
353 | if spr is None: |
354 | spr = self.makeSPR() |
355 | @@ -1323,12 +1382,13 @@ class TestArchSpecificPublicationsCache(TestCaseWithFactory): |
356 | bpb = self.factory.makeBinaryPackageBuild( |
357 | source_package_release=spr, distroarchseries=das) |
358 | bpr = self.factory.makeBinaryPackageRelease( |
359 | - build=bpb, architecturespecific=arch_specific) |
360 | + build=bpb, binpackageformat=binpackageformat, |
361 | + architecturespecific=arch_specific) |
362 | return removeSecurityProxy( |
363 | self.factory.makeBinaryPackagePublishingHistory( |
364 | binarypackagerelease=bpr, archive=archive, |
365 | distroarchseries=das, pocket=PackagePublishingPocket.UPDATES, |
366 | - status=PackagePublishingStatus.PUBLISHED)) |
367 | + status=PackagePublishingStatus.PUBLISHED, channel=channel)) |
368 | |
369 | def test_getKey_is_consistent_and_distinguishing(self): |
370 | # getKey consistently returns the same key for the same BPPH, |
371 | @@ -1351,14 +1411,19 @@ class TestArchSpecificPublicationsCache(TestCaseWithFactory): |
372 | spr, arch_specific=False, archive=dependent.archive, |
373 | distroseries=dependent.distroseries) |
374 | bpph2 = self.makeBPPH(arch_specific=False) |
375 | + bpph3 = self.makeBPPH( |
376 | + arch_specific=False, binpackageformat=BinaryPackageFormat.WHL, |
377 | + channel="edge") |
378 | cache = self.makeCache() |
379 | self.assertEqual( |
380 | - [True, True, False, False], |
381 | + [True, True, False, False, False, False], |
382 | [ |
383 | cache.hasArchSpecificPublications(bpph1), |
384 | cache.hasArchSpecificPublications(bpph1), |
385 | cache.hasArchSpecificPublications(bpph2), |
386 | cache.hasArchSpecificPublications(bpph2), |
387 | + cache.hasArchSpecificPublications(bpph3), |
388 | + cache.hasArchSpecificPublications(bpph3), |
389 | ]) |
390 | |
391 | def test_hasArchSpecificPublications_caches_results(self): |
392 | diff --git a/lib/lp/code/interfaces/cibuild.py b/lib/lp/code/interfaces/cibuild.py |
393 | index 6146dfe..fc7a197 100644 |
394 | --- a/lib/lp/code/interfaces/cibuild.py |
395 | +++ b/lib/lp/code/interfaces/cibuild.py |
396 | @@ -177,6 +177,14 @@ class ICIBuildView(IPackageBuildView, IPrivacy): |
397 | :return: A collection of URLs for this build. |
398 | """ |
399 | |
400 | + def createBinaryPackageRelease( |
401 | + binarypackagename, version, summary, description, binpackageformat, |
402 | + architecturespecific, installedsize=None, homepage=None): |
403 | + """Create and return a `BinaryPackageRelease` for this CI build. |
404 | + |
405 | + The new binary package release will be linked to this build. |
406 | + """ |
407 | + |
408 | |
409 | class ICIBuildEdit(IBuildFarmJobEdit): |
410 | """`ICIBuild` methods that require launchpad.Edit.""" |
411 | diff --git a/lib/lp/code/model/cibuild.py b/lib/lp/code/model/cibuild.py |
412 | index d789a95..4e1ddbd 100644 |
413 | --- a/lib/lp/code/model/cibuild.py |
414 | +++ b/lib/lp/code/model/cibuild.py |
415 | @@ -85,6 +85,7 @@ from lp.services.macaroons.interfaces import ( |
416 | ) |
417 | from lp.services.macaroons.model import MacaroonIssuerBase |
418 | from lp.services.propertycache import cachedproperty |
419 | +from lp.soyuz.model.binarypackagerelease import BinaryPackageRelease |
420 | from lp.soyuz.model.distroarchseries import DistroArchSeries |
421 | |
422 | |
423 | @@ -465,6 +466,18 @@ class CIBuild(PackageBuildMixin, StormBase): |
424 | """See `IPackageBuild`.""" |
425 | # We don't currently send any notifications. |
426 | |
427 | + def createBinaryPackageRelease( |
428 | + self, binarypackagename, version, summary, description, |
429 | + binpackageformat, architecturespecific, installedsize=None, |
430 | + homepage=None): |
431 | + """See `ICIBuild`.""" |
432 | + return BinaryPackageRelease( |
433 | + ci_build=self, binarypackagename=binarypackagename, |
434 | + version=version, summary=summary, description=description, |
435 | + binpackageformat=binpackageformat, |
436 | + architecturespecific=architecturespecific, |
437 | + installedsize=installedsize, homepage=homepage) |
438 | + |
439 | |
440 | @implementer(ICIBuildSet) |
441 | class CIBuildSet(SpecificBuildFarmJobSourceMixin): |
442 | diff --git a/lib/lp/code/model/tests/test_cibuild.py b/lib/lp/code/model/tests/test_cibuild.py |
443 | index 899b09b..b7e0ca5 100644 |
444 | --- a/lib/lp/code/model/tests/test_cibuild.py |
445 | +++ b/lib/lp/code/model/tests/test_cibuild.py |
446 | @@ -70,6 +70,7 @@ from lp.services.macaroons.interfaces import IMacaroonIssuer |
447 | from lp.services.macaroons.testing import MacaroonTestMixin |
448 | from lp.services.propertycache import clear_property_cache |
449 | from lp.services.webapp.interfaces import OAuthPermission |
450 | +from lp.soyuz.enums import BinaryPackageFormat |
451 | from lp.testing import ( |
452 | ANONYMOUS, |
453 | api_url, |
454 | @@ -399,6 +400,24 @@ class TestCIBuild(TestCaseWithFactory): |
455 | commit_sha1=build.commit_sha1, |
456 | ci_build=build)) |
457 | |
458 | + def test_createBinaryPackageRelease(self): |
459 | + build = self.factory.makeCIBuild() |
460 | + bpn = self.factory.makeBinaryPackageName() |
461 | + bpr = build.createBinaryPackageRelease( |
462 | + bpn, "1.0", "test summary", "test description", |
463 | + BinaryPackageFormat.WHL, False, installedsize=1024, |
464 | + homepage="https://example.com/") |
465 | + self.assertThat(bpr, MatchesStructure( |
466 | + binarypackagename=Equals(bpn), |
467 | + version=Equals("1.0"), |
468 | + summary=Equals("test summary"), |
469 | + description=Equals("test description"), |
470 | + binpackageformat=Equals(BinaryPackageFormat.WHL), |
471 | + architecturespecific=Is(False), |
472 | + installedsize=Equals(1024), |
473 | + homepage=Equals("https://example.com/"), |
474 | + )) |
475 | + |
476 | |
477 | class TestCIBuildSet(TestCaseWithFactory): |
478 | |
479 | diff --git a/lib/lp/soyuz/adapters/packagelocation.py b/lib/lp/soyuz/adapters/packagelocation.py |
480 | index 6644f1e..2cf1cb4 100644 |
481 | --- a/lib/lp/soyuz/adapters/packagelocation.py |
482 | +++ b/lib/lp/soyuz/adapters/packagelocation.py |
483 | @@ -29,9 +29,10 @@ class PackageLocation: |
484 | pocket = None |
485 | component = None |
486 | packagesets = None |
487 | + channel = None |
488 | |
489 | def __init__(self, archive, distribution, distroseries, pocket, |
490 | - component=None, packagesets=None): |
491 | + component=None, packagesets=None, channel=None): |
492 | """Initialize the PackageLocation from the given parameters.""" |
493 | self.archive = archive |
494 | self.distribution = distribution |
495 | @@ -39,6 +40,7 @@ class PackageLocation: |
496 | self.pocket = pocket |
497 | self.component = component |
498 | self.packagesets = packagesets or [] |
499 | + self.channel = channel |
500 | |
501 | def __eq__(self, other): |
502 | if (self.distribution == other.distribution and |
503 | @@ -46,10 +48,22 @@ class PackageLocation: |
504 | self.distroseries == other.distroseries and |
505 | self.component == other.component and |
506 | self.pocket == other.pocket and |
507 | - self.packagesets == other.packagesets): |
508 | + self.packagesets == other.packagesets and |
509 | + self.channel == other.channel): |
510 | return True |
511 | return False |
512 | |
513 | + def __hash__(self): |
514 | + return hash(( |
515 | + self.archive, |
516 | + self.distribution, |
517 | + self.distroseries, |
518 | + self.pocket, |
519 | + self.component, |
520 | + None if self.packagesets is None else tuple(self.packagesets), |
521 | + self.channel, |
522 | + )) |
523 | + |
524 | def __str__(self): |
525 | result = '%s: %s-%s' % ( |
526 | self.archive.reference, self.distroseries.name, self.pocket.name) |
527 | @@ -61,6 +75,9 @@ class PackageLocation: |
528 | result += ' [%s]' % ( |
529 | ", ".join([str(p.name) for p in self.packagesets]),) |
530 | |
531 | + if self.channel is not None: |
532 | + result += ' {%s}' % self.channel |
533 | + |
534 | return result |
535 | |
536 | |
537 | @@ -70,7 +87,7 @@ class PackageLocationError(Exception): |
538 | |
539 | def build_package_location(distribution_name, suite=None, purpose=None, |
540 | person_name=None, archive_name=None, |
541 | - packageset_names=None): |
542 | + packageset_names=None, channel=None): |
543 | """Convenience function to build PackageLocation objects.""" |
544 | |
545 | # XXX kiko 2007-10-24: |
546 | @@ -143,6 +160,10 @@ def build_package_location(distribution_name, suite=None, purpose=None, |
547 | distroseries = distribution.currentseries |
548 | pocket = PackagePublishingPocket.RELEASE |
549 | |
550 | + if pocket != PackagePublishingPocket.RELEASE and channel is not None: |
551 | + raise PackageLocationError( |
552 | + "Channels may only be used with the RELEASE pocket.") |
553 | + |
554 | packagesets = [] |
555 | if packageset_names: |
556 | packageset_set = getUtility(IPackagesetSet) |
557 | @@ -155,5 +176,6 @@ def build_package_location(distribution_name, suite=None, purpose=None, |
558 | "Could not find packageset %s" % err) |
559 | packagesets.append(packageset) |
560 | |
561 | - return PackageLocation(archive, distribution, distroseries, pocket, |
562 | - packagesets=packagesets) |
563 | + return PackageLocation( |
564 | + archive, distribution, distroseries, pocket, |
565 | + packagesets=packagesets, channel=channel) |
566 | diff --git a/lib/lp/soyuz/adapters/tests/test_packagelocation.py b/lib/lp/soyuz/adapters/tests/test_packagelocation.py |
567 | index 3d45f14..9bbef24 100644 |
568 | --- a/lib/lp/soyuz/adapters/tests/test_packagelocation.py |
569 | +++ b/lib/lp/soyuz/adapters/tests/test_packagelocation.py |
570 | @@ -22,11 +22,12 @@ class TestPackageLocation(TestCaseWithFactory): |
571 | |
572 | def getPackageLocation(self, distribution_name='ubuntu', suite=None, |
573 | purpose=None, person_name=None, |
574 | - archive_name=None, packageset_names=None): |
575 | + archive_name=None, packageset_names=None, |
576 | + channel=None): |
577 | """Use a helper method to setup a `PackageLocation` object.""" |
578 | return build_package_location( |
579 | distribution_name, suite, purpose, person_name, archive_name, |
580 | - packageset_names=packageset_names) |
581 | + packageset_names=packageset_names, channel=channel) |
582 | |
583 | def testSetupLocationForCOPY(self): |
584 | """`PackageLocation` for COPY archives.""" |
585 | @@ -150,6 +151,15 @@ class TestPackageLocation(TestCaseWithFactory): |
586 | distribution_name='debian', |
587 | packageset_names=[packageset_name, "unknown"]) |
588 | |
589 | + def test_build_package_location_with_channel_outside_release_pocket(self): |
590 | + """It doesn't make sense to use non-RELEASE pockets with channels.""" |
591 | + self.assertRaisesWithContent( |
592 | + PackageLocationError, |
593 | + "Channels may only be used with the RELEASE pocket.", |
594 | + self.getPackageLocation, |
595 | + suite="warty-security", |
596 | + channel="stable") |
597 | + |
598 | def testSetupLocationPPANotMatchingDistribution(self): |
599 | """`PackageLocationError` is raised when PPA does not match the |
600 | distribution.""" |
601 | @@ -214,6 +224,14 @@ class TestPackageLocation(TestCaseWithFactory): |
602 | self.assertEqual( |
603 | location_ubuntu_hoary, location_ubuntu_hoary_again) |
604 | |
605 | + def testCompareChannels(self): |
606 | + location_ubuntu_hoary = self.getPackageLocation(channel="stable") |
607 | + location_ubuntu_hoary_again = self.getPackageLocation(channel="edge") |
608 | + self.assertNotEqual(location_ubuntu_hoary, location_ubuntu_hoary_again) |
609 | + |
610 | + location_ubuntu_hoary_again.channel = "stable" |
611 | + self.assertEqual(location_ubuntu_hoary, location_ubuntu_hoary_again) |
612 | + |
613 | def testRepresentation(self): |
614 | """Check if PackageLocation is represented correctly.""" |
615 | location_ubuntu_hoary = self.getPackageLocation() |
616 | diff --git a/lib/lp/soyuz/enums.py b/lib/lp/soyuz/enums.py |
617 | index d60ea48..9953f43 100644 |
618 | --- a/lib/lp/soyuz/enums.py |
619 | +++ b/lib/lp/soyuz/enums.py |
620 | @@ -208,6 +208,13 @@ class BinaryPackageFileType(DBEnumeratedType): |
621 | build environment. |
622 | """) |
623 | |
624 | + WHL = DBItem(6, """ |
625 | + Python Wheel |
626 | + |
627 | + The "wheel" binary package format for Python, originally defined in |
628 | + U{https://peps.python.org/pep-0427/}. |
629 | + """) |
630 | + |
631 | |
632 | class BinaryPackageFormat(DBEnumeratedType): |
633 | """Binary Package Format |
634 | @@ -251,6 +258,13 @@ class BinaryPackageFormat(DBEnumeratedType): |
635 | This is the binary package format used for shipping debug symbols |
636 | in Ubuntu and similar distributions.""") |
637 | |
638 | + WHL = DBItem(6, """ |
639 | + Python Wheel |
640 | + |
641 | + The "wheel" binary package format for Python, originally defined in |
642 | + U{https://peps.python.org/pep-0427/}. |
643 | + """) |
644 | + |
645 | |
646 | class PackageCopyPolicy(DBEnumeratedType): |
647 | """Package copying policy. |
648 | diff --git a/lib/lp/soyuz/interfaces/publishing.py b/lib/lp/soyuz/interfaces/publishing.py |
649 | index a1cfeab..21a3cac 100644 |
650 | --- a/lib/lp/soyuz/interfaces/publishing.py |
651 | +++ b/lib/lp/soyuz/interfaces/publishing.py |
652 | @@ -50,7 +50,6 @@ from zope.schema import ( |
653 | Date, |
654 | Datetime, |
655 | Int, |
656 | - List, |
657 | Text, |
658 | TextLine, |
659 | ) |
660 | @@ -269,9 +268,8 @@ class ISourcePackagePublishingHistoryPublic(IPublishingView): |
661 | vocabulary=PackagePublishingPocket, |
662 | required=True, readonly=True, |
663 | )) |
664 | - channel = List( |
665 | - value_type=TextLine(), title=_("Channel"), |
666 | - required=False, readonly=False, |
667 | + channel = TextLine( |
668 | + title=_("Channel"), required=False, readonly=False, |
669 | description=_( |
670 | "The channel into which this entry is published " |
671 | "(only for archives published using Artifactory)")) |
672 | @@ -700,9 +698,8 @@ class IBinaryPackagePublishingHistoryPublic(IPublishingView): |
673 | vocabulary=PackagePublishingPocket, |
674 | required=True, readonly=True, |
675 | )) |
676 | - channel = List( |
677 | - value_type=TextLine(), title=_("Channel"), |
678 | - required=False, readonly=False, |
679 | + channel = TextLine( |
680 | + title=_("Channel"), required=False, readonly=False, |
681 | description=_( |
682 | "The channel into which this entry is published " |
683 | "(only for archives published using Artifactory)")) |
684 | @@ -970,7 +967,8 @@ class IPublishingSet(Interface): |
685 | def newSourcePublication(archive, sourcepackagerelease, distroseries, |
686 | component, section, pocket, ancestor, |
687 | create_dsd_job=True, copied_from_archive=None, |
688 | - creator=None, sponsor=None, packageupload=None): |
689 | + creator=None, sponsor=None, packageupload=None, |
690 | + channel=None): |
691 | """Create a new `SourcePackagePublishingHistory`. |
692 | |
693 | :param archive: An `IArchive` |
694 | diff --git a/lib/lp/soyuz/model/binarypackagerelease.py b/lib/lp/soyuz/model/binarypackagerelease.py |
695 | index 25cae48..7134895 100644 |
696 | --- a/lib/lp/soyuz/model/binarypackagerelease.py |
697 | +++ b/lib/lp/soyuz/model/binarypackagerelease.py |
698 | @@ -157,6 +157,8 @@ class BinaryPackageRelease(SQLBase): |
699 | determined_filetype = BinaryPackageFileType.UDEB |
700 | elif file.filename.endswith(".ddeb"): |
701 | determined_filetype = BinaryPackageFileType.DDEB |
702 | + elif file.filename.endswith(".whl"): |
703 | + determined_filetype = BinaryPackageFileType.WHL |
704 | else: |
705 | raise AssertionError( |
706 | 'Unsupported file type: %s' % file.filename) |
707 | diff --git a/lib/lp/soyuz/model/publishing.py b/lib/lp/soyuz/model/publishing.py |
708 | index cdab414..8a397b6 100644 |
709 | --- a/lib/lp/soyuz/model/publishing.py |
710 | +++ b/lib/lp/soyuz/model/publishing.py |
711 | @@ -12,6 +12,7 @@ __all__ = [ |
712 | |
713 | from collections import defaultdict |
714 | from datetime import datetime |
715 | +import json |
716 | from operator import ( |
717 | attrgetter, |
718 | itemgetter, |
719 | @@ -20,6 +21,7 @@ from pathlib import Path |
720 | import sys |
721 | |
722 | import pytz |
723 | +from storm.databases.postgres import JSON |
724 | from storm.expr import ( |
725 | And, |
726 | Cast, |
727 | @@ -31,10 +33,6 @@ from storm.expr import ( |
728 | Sum, |
729 | ) |
730 | from storm.info import ClassAlias |
731 | -from storm.properties import ( |
732 | - List, |
733 | - Unicode, |
734 | - ) |
735 | from storm.store import Store |
736 | from storm.zope import IResultSet |
737 | from storm.zope.interfaces import ISQLObjectResultSet |
738 | @@ -51,6 +49,10 @@ from lp.registry.interfaces.person import validate_public_person |
739 | from lp.registry.interfaces.pocket import PackagePublishingPocket |
740 | from lp.registry.interfaces.sourcepackage import SourcePackageType |
741 | from lp.registry.model.sourcepackagename import SourcePackageName |
742 | +from lp.services.channels import ( |
743 | + channel_list_to_string, |
744 | + channel_string_to_list, |
745 | + ) |
746 | from lp.services.database import bulk |
747 | from lp.services.database.constants import UTC_NOW |
748 | from lp.services.database.datetimecol import UtcDateTimeCol |
749 | @@ -267,7 +269,7 @@ class SourcePackagePublishingHistory(SQLBase, ArchivePublisherBase): |
750 | pocket = DBEnum(name='pocket', enum=PackagePublishingPocket, |
751 | default=PackagePublishingPocket.RELEASE, |
752 | allow_none=False) |
753 | - channel = List(name="channel", type=Unicode(), allow_none=True) |
754 | + _channel = JSON(name="channel", allow_none=True) |
755 | archive = ForeignKey(dbName="archive", foreignKey="Archive", notNull=True) |
756 | copied_from_archive = ForeignKey( |
757 | dbName="copied_from_archive", foreignKey="Archive", notNull=False) |
758 | @@ -315,6 +317,13 @@ class SourcePackagePublishingHistory(SQLBase, ArchivePublisherBase): |
759 | self.distroseries.setNewerDistroSeriesVersions([self]) |
760 | return get_property_cache(self).newer_distroseries_version |
761 | |
762 | + @property |
763 | + def channel(self): |
764 | + """See `ISourcePackagePublishingHistory`.""" |
765 | + if self._channel is None: |
766 | + return None |
767 | + return channel_list_to_string(*self._channel) |
768 | + |
769 | def getPublishedBinaries(self): |
770 | """See `ISourcePackagePublishingHistory`.""" |
771 | publishing_set = getUtility(IPublishingSet) |
772 | @@ -538,7 +547,8 @@ class SourcePackagePublishingHistory(SQLBase, ArchivePublisherBase): |
773 | component=new_component, |
774 | section=new_section, |
775 | creator=creator, |
776 | - archive=self.archive) |
777 | + archive=self.archive, |
778 | + channel=self.channel) |
779 | |
780 | def copyTo(self, distroseries, pocket, archive, override=None, |
781 | create_dsd_job=True, creator=None, sponsor=None, |
782 | @@ -564,7 +574,8 @@ class SourcePackagePublishingHistory(SQLBase, ArchivePublisherBase): |
783 | creator=creator, |
784 | sponsor=sponsor, |
785 | copied_from_archive=self.archive, |
786 | - packageupload=packageupload) |
787 | + packageupload=packageupload, |
788 | + channel=self.channel) |
789 | |
790 | def getStatusSummaryForBuilds(self): |
791 | """See `ISourcePackagePublishingHistory`.""" |
792 | @@ -685,7 +696,7 @@ class BinaryPackagePublishingHistory(SQLBase, ArchivePublisherBase): |
793 | datemadepending = UtcDateTimeCol(default=None) |
794 | dateremoved = UtcDateTimeCol(default=None) |
795 | pocket = DBEnum(name='pocket', enum=PackagePublishingPocket) |
796 | - channel = List(name="channel", type=Unicode(), allow_none=True) |
797 | + _channel = JSON(name="channel", allow_none=True) |
798 | archive = ForeignKey(dbName="archive", foreignKey="Archive", notNull=True) |
799 | copied_from_archive = ForeignKey( |
800 | dbName="copied_from_archive", foreignKey="Archive", notNull=False) |
801 | @@ -779,6 +790,13 @@ class BinaryPackagePublishingHistory(SQLBase, ArchivePublisherBase): |
802 | distroseries.name, |
803 | self.distroarchseries.architecturetag) |
804 | |
805 | + @property |
806 | + def channel(self): |
807 | + """See `ISourcePackagePublishingHistory`.""" |
808 | + if self._channel is None: |
809 | + return None |
810 | + return channel_list_to_string(*self._channel) |
811 | + |
812 | def getDownloadCount(self): |
813 | """See `IBinaryPackagePublishingHistory`.""" |
814 | return self.archive.getPackageDownloadTotal(self.binarypackagerelease) |
815 | @@ -836,21 +854,26 @@ class BinaryPackagePublishingHistory(SQLBase, ArchivePublisherBase): |
816 | dominant.distroarchseries.architecturetag)) |
817 | |
818 | dominant_build = dominant.binarypackagerelease.build |
819 | - distroarchseries = dominant_build.distro_arch_series |
820 | - if logger is not None: |
821 | - logger.debug( |
822 | - "The %s build of %s has been judged as superseded by the " |
823 | - "build of %s. Arch-specific == %s" % ( |
824 | - distroarchseries.architecturetag, |
825 | - self.binarypackagerelease.title, |
826 | - dominant_build.source_package_release.title, |
827 | - self.binarypackagerelease.architecturespecific)) |
828 | - # Binary package releases are superseded by the new build, |
829 | - # not the new binary package release. This is because |
830 | - # there may not *be* a new matching binary package - |
831 | - # source packages can change the binaries they build |
832 | - # between releases. |
833 | - self.supersededby = dominant_build |
834 | + # XXX cjwatson 2022-05-01: We can't currently dominate with CI |
835 | + # builds, since supersededby is a reference to a BPB. Just |
836 | + # leave supersededby unset in that case for now, which isn't |
837 | + # ideal but will work well enough. |
838 | + if dominant_build is not None: |
839 | + distroarchseries = dominant_build.distro_arch_series |
840 | + if logger is not None: |
841 | + logger.debug( |
842 | + "The %s build of %s has been judged as superseded by " |
843 | + "the build of %s. Arch-specific == %s" % ( |
844 | + distroarchseries.architecturetag, |
845 | + self.binarypackagerelease.title, |
846 | + dominant_build.source_package_release.title, |
847 | + self.binarypackagerelease.architecturespecific)) |
848 | + # Binary package releases are superseded by the new build, |
849 | + # not the new binary package release. This is because |
850 | + # there may not *be* a new matching binary package - |
851 | + # source packages can change the binaries they build |
852 | + # between releases. |
853 | + self.supersededby = dominant_build |
854 | |
855 | debug = getUtility(IPublishingSet).findCorrespondingDDEBPublications( |
856 | [self]) |
857 | @@ -941,7 +964,8 @@ class BinaryPackagePublishingHistory(SQLBase, ArchivePublisherBase): |
858 | priority=new_priority, |
859 | creator=creator, |
860 | archive=debug.archive, |
861 | - phased_update_percentage=new_phased_update_percentage) |
862 | + phased_update_percentage=new_phased_update_percentage, |
863 | + _channel=removeSecurityProxy(debug)._channel) |
864 | |
865 | # Append the modified package publishing entry |
866 | return BinaryPackagePublishingHistory( |
867 | @@ -957,7 +981,8 @@ class BinaryPackagePublishingHistory(SQLBase, ArchivePublisherBase): |
868 | priority=new_priority, |
869 | archive=self.archive, |
870 | creator=creator, |
871 | - phased_update_percentage=new_phased_update_percentage) |
872 | + phased_update_percentage=new_phased_update_percentage, |
873 | + _channel=self._channel) |
874 | |
875 | def copyTo(self, distroseries, pocket, archive): |
876 | """See `BinaryPackagePublishingHistory`.""" |
877 | @@ -1086,10 +1111,15 @@ class PublishingSet: |
878 | """Utilities for manipulating publications in batches.""" |
879 | |
880 | def publishBinaries(self, archive, distroseries, pocket, binaries, |
881 | - copied_from_archives=None): |
882 | + copied_from_archives=None, channel=None): |
883 | """See `IPublishingSet`.""" |
884 | if copied_from_archives is None: |
885 | copied_from_archives = {} |
886 | + if channel is not None: |
887 | + if pocket != PackagePublishingPocket.RELEASE: |
888 | + raise AssertionError( |
889 | + "Channel publications must be in the RELEASE pocket") |
890 | + channel = channel_string_to_list(channel) |
891 | # Expand the dict of binaries into a list of tuples including the |
892 | # architecture. |
893 | if distroseries.distribution != archive.distribution: |
894 | @@ -1124,6 +1154,9 @@ class PublishingSet: |
895 | BinaryPackageRelease.binarypackagenameID, |
896 | BinaryPackageRelease.version), |
897 | BinaryPackagePublishingHistory.pocket == pocket, |
898 | + Not(IsDistinctFrom( |
899 | + BinaryPackagePublishingHistory._channel, |
900 | + json.dumps(channel) if channel is not None else None)), |
901 | BinaryPackagePublishingHistory.status.is_in( |
902 | active_publishing_status), |
903 | BinaryPackageRelease.id == |
904 | @@ -1141,12 +1174,13 @@ class PublishingSet: |
905 | BPPH = BinaryPackagePublishingHistory |
906 | return bulk.create( |
907 | (BPPH.archive, BPPH.copied_from_archive, |
908 | - BPPH.distroarchseries, BPPH.pocket, |
909 | + BPPH.distroarchseries, BPPH.pocket, BPPH._channel, |
910 | BPPH.binarypackagerelease, BPPH.binarypackagename, |
911 | + BPPH._binarypackageformat, |
912 | BPPH.component, BPPH.section, BPPH.priority, |
913 | BPPH.phased_update_percentage, BPPH.status, BPPH.datecreated), |
914 | - [(archive, copied_from_archives.get(bpr), das, pocket, bpr, |
915 | - bpr.binarypackagename, |
916 | + [(archive, copied_from_archives.get(bpr), das, pocket, channel, |
917 | + bpr, bpr.binarypackagename, bpr.binpackageformat, |
918 | get_component(archive, das.distroseries, component), |
919 | section, priority, phased_update_percentage, |
920 | PackagePublishingStatus.PENDING, UTC_NOW) |
921 | @@ -1156,7 +1190,7 @@ class PublishingSet: |
922 | get_objects=True) |
923 | |
924 | def copyBinaries(self, archive, distroseries, pocket, bpphs, policy=None, |
925 | - source_override=None): |
926 | + source_override=None, channel=None): |
927 | """See `IPublishingSet`.""" |
928 | from lp.soyuz.adapters.overrides import BinaryOverride |
929 | if distroseries.distribution != archive.distribution: |
930 | @@ -1228,13 +1262,14 @@ class PublishingSet: |
931 | bpph.binarypackagerelease: bpph.archive for bpph in bpphs} |
932 | return self.publishBinaries( |
933 | archive, distroseries, pocket, with_overrides, |
934 | - copied_from_archives) |
935 | + copied_from_archives, channel=channel) |
936 | |
937 | def newSourcePublication(self, archive, sourcepackagerelease, |
938 | distroseries, component, section, pocket, |
939 | ancestor=None, create_dsd_job=True, |
940 | copied_from_archive=None, |
941 | - creator=None, sponsor=None, packageupload=None): |
942 | + creator=None, sponsor=None, packageupload=None, |
943 | + channel=None): |
944 | """See `IPublishingSet`.""" |
945 | # Circular import. |
946 | from lp.registry.model.distributionsourcepackage import ( |
947 | @@ -1246,6 +1281,15 @@ class PublishingSet: |
948 | "Series distribution %s doesn't match archive distribution %s." |
949 | % (distroseries.distribution.name, archive.distribution.name)) |
950 | |
951 | + if channel is not None: |
952 | + if sourcepackagerelease.format == SourcePackageType.DPKG: |
953 | + raise AssertionError( |
954 | + "Can't publish dpkg source packages to a channel") |
955 | + if pocket != PackagePublishingPocket.RELEASE: |
956 | + raise AssertionError( |
957 | + "Channel publications must be in the RELEASE pocket") |
958 | + channel = channel_string_to_list(channel) |
959 | + |
960 | pub = SourcePackagePublishingHistory( |
961 | distroseries=distroseries, |
962 | pocket=pocket, |
963 | @@ -1261,7 +1305,8 @@ class PublishingSet: |
964 | ancestor=ancestor, |
965 | creator=creator, |
966 | sponsor=sponsor, |
967 | - packageupload=packageupload) |
968 | + packageupload=packageupload, |
969 | + _channel=channel) |
970 | DistributionSourcePackage.ensure(pub) |
971 | |
972 | if create_dsd_job and archive == distroseries.main_archive: |
973 | diff --git a/lib/lp/soyuz/tests/test_publishing.py b/lib/lp/soyuz/tests/test_publishing.py |
974 | index 41cdf25..7d91f43 100644 |
975 | --- a/lib/lp/soyuz/tests/test_publishing.py |
976 | +++ b/lib/lp/soyuz/tests/test_publishing.py |
977 | @@ -36,6 +36,7 @@ from lp.registry.interfaces.sourcepackage import ( |
978 | SourcePackageUrgency, |
979 | ) |
980 | from lp.registry.interfaces.sourcepackagename import ISourcePackageNameSet |
981 | +from lp.services.channels import channel_string_to_list |
982 | from lp.services.config import config |
983 | from lp.services.database.constants import UTC_NOW |
984 | from lp.services.librarian.interfaces import ILibraryFileAliasSet |
985 | @@ -212,7 +213,8 @@ class SoyuzTestPublisher: |
986 | build_conflicts_indep=None, |
987 | dsc_maintainer_rfc822='Foo Bar <foo@bar.com>', |
988 | maintainer=None, creator=None, date_uploaded=UTC_NOW, |
989 | - spr_only=False, user_defined_fields=None): |
990 | + spr_only=False, user_defined_fields=None, |
991 | + format=SourcePackageType.DPKG, channel=None): |
992 | """Return a mock source publishing record. |
993 | |
994 | if spr_only is specified, the source is not published and the |
995 | @@ -238,7 +240,7 @@ class SoyuzTestPublisher: |
996 | |
997 | spr = distroseries.createUploadedSourcePackageRelease( |
998 | sourcepackagename=spn, |
999 | - format=SourcePackageType.DPKG, |
1000 | + format=format, |
1001 | maintainer=maintainer, |
1002 | creator=creator, |
1003 | component=component, |
1004 | @@ -289,6 +291,8 @@ class SoyuzTestPublisher: |
1005 | datepublished = UTC_NOW |
1006 | else: |
1007 | datepublished = None |
1008 | + if channel is not None: |
1009 | + channel = channel_string_to_list(channel) |
1010 | |
1011 | spph = SourcePackagePublishingHistory( |
1012 | distroseries=distroseries, |
1013 | @@ -304,7 +308,8 @@ class SoyuzTestPublisher: |
1014 | scheduleddeletiondate=scheduleddeletiondate, |
1015 | pocket=pocket, |
1016 | archive=archive, |
1017 | - creator=creator) |
1018 | + creator=creator, |
1019 | + _channel=channel) |
1020 | |
1021 | return spph |
1022 | |
1023 | @@ -328,7 +333,8 @@ class SoyuzTestPublisher: |
1024 | builder=None, |
1025 | component='main', |
1026 | phased_update_percentage=None, |
1027 | - with_debug=False, user_defined_fields=None): |
1028 | + with_debug=False, user_defined_fields=None, |
1029 | + channel=None): |
1030 | """Return a list of binary publishing records.""" |
1031 | if distroseries is None: |
1032 | distroseries = self.distroseries |
1033 | @@ -366,7 +372,7 @@ class SoyuzTestPublisher: |
1034 | pub_binaries += self.publishBinaryInArchive( |
1035 | binarypackagerelease_ddeb, archive, status, |
1036 | pocket, scheduleddeletiondate, dateremoved, |
1037 | - phased_update_percentage) |
1038 | + phased_update_percentage, channel=channel) |
1039 | else: |
1040 | binarypackagerelease_ddeb = None |
1041 | |
1042 | @@ -378,7 +384,8 @@ class SoyuzTestPublisher: |
1043 | user_defined_fields=user_defined_fields) |
1044 | pub_binaries += self.publishBinaryInArchive( |
1045 | binarypackagerelease, archive, status, pocket, |
1046 | - scheduleddeletiondate, dateremoved, phased_update_percentage) |
1047 | + scheduleddeletiondate, dateremoved, phased_update_percentage, |
1048 | + channel=channel) |
1049 | published_binaries.extend(pub_binaries) |
1050 | package_upload = self.addPackageUpload( |
1051 | archive, distroseries, pocket, |
1052 | @@ -476,7 +483,7 @@ class SoyuzTestPublisher: |
1053 | status=PackagePublishingStatus.PENDING, |
1054 | pocket=PackagePublishingPocket.RELEASE, |
1055 | scheduleddeletiondate=None, dateremoved=None, |
1056 | - phased_update_percentage=None): |
1057 | + phased_update_percentage=None, channel=None): |
1058 | """Return the corresponding BinaryPackagePublishingHistory.""" |
1059 | distroarchseries = binarypackagerelease.build.distro_arch_series |
1060 | |
1061 | @@ -485,6 +492,8 @@ class SoyuzTestPublisher: |
1062 | archs = [distroarchseries] |
1063 | else: |
1064 | archs = distroarchseries.distroseries.architectures |
1065 | + if channel is not None: |
1066 | + channel = channel_string_to_list(channel) |
1067 | |
1068 | pub_binaries = [] |
1069 | for arch in archs: |
1070 | @@ -502,7 +511,8 @@ class SoyuzTestPublisher: |
1071 | datecreated=UTC_NOW, |
1072 | pocket=pocket, |
1073 | archive=archive, |
1074 | - phased_update_percentage=phased_update_percentage) |
1075 | + phased_update_percentage=phased_update_percentage, |
1076 | + _channel=channel) |
1077 | if status == PackagePublishingStatus.PUBLISHED: |
1078 | pub.datepublished = UTC_NOW |
1079 | pub_binaries.append(pub) |
1080 | @@ -1583,19 +1593,24 @@ class TestPublishBinaries(TestCaseWithFactory): |
1081 | |
1082 | layer = LaunchpadZopelessLayer |
1083 | |
1084 | - def makeArgs(self, bprs, distroseries, archive=None): |
1085 | + def makeArgs(self, bprs, distroseries, archive=None, channel=None): |
1086 | """Create a dict of arguments for publishBinaries.""" |
1087 | if archive is None: |
1088 | archive = distroseries.main_archive |
1089 | - return { |
1090 | + args = { |
1091 | 'archive': archive, |
1092 | 'distroseries': distroseries, |
1093 | - 'pocket': PackagePublishingPocket.BACKPORTS, |
1094 | + 'pocket': ( |
1095 | + PackagePublishingPocket.BACKPORTS if channel is None |
1096 | + else PackagePublishingPocket.RELEASE), |
1097 | 'binaries': { |
1098 | bpr: (self.factory.makeComponent(), |
1099 | self.factory.makeSection(), |
1100 | PackagePublishingPriority.REQUIRED, 50) for bpr in bprs}, |
1101 | } |
1102 | + if channel is not None: |
1103 | + args['channel'] = channel |
1104 | + return args |
1105 | |
1106 | def test_architecture_dependent(self): |
1107 | # Architecture-dependent binaries get created as PENDING in the |
1108 | @@ -1614,8 +1629,8 @@ class TestPublishBinaries(TestCaseWithFactory): |
1109 | overrides = args['binaries'][bpr] |
1110 | self.assertEqual(bpr, bpph.binarypackagerelease) |
1111 | self.assertEqual( |
1112 | - (args['archive'], target_das, args['pocket']), |
1113 | - (bpph.archive, bpph.distroarchseries, bpph.pocket)) |
1114 | + (args['archive'], target_das, args['pocket'], None), |
1115 | + (bpph.archive, bpph.distroarchseries, bpph.pocket, bpph.channel)) |
1116 | self.assertEqual( |
1117 | overrides, |
1118 | (bpph.component, bpph.section, bpph.priority, |
1119 | @@ -1670,30 +1685,59 @@ class TestPublishBinaries(TestCaseWithFactory): |
1120 | args['pocket'] = PackagePublishingPocket.RELEASE |
1121 | [another_bpph] = getUtility(IPublishingSet).publishBinaries(**args) |
1122 | |
1123 | + def test_channel(self): |
1124 | + bpr = self.factory.makeBinaryPackageRelease( |
1125 | + binpackageformat=BinaryPackageFormat.WHL) |
1126 | + target_das = self.factory.makeDistroArchSeries() |
1127 | + args = self.makeArgs([bpr], target_das.distroseries, channel="stable") |
1128 | + [bpph] = getUtility(IPublishingSet).publishBinaries(**args) |
1129 | + self.assertEqual(bpr, bpph.binarypackagerelease) |
1130 | + self.assertEqual( |
1131 | + (args["archive"], target_das, args["pocket"], args["channel"]), |
1132 | + (bpph.archive, bpph.distroarchseries, bpph.pocket, bpph.channel)) |
1133 | + self.assertEqual(PackagePublishingStatus.PENDING, bpph.status) |
1134 | + |
1135 | + def test_does_not_duplicate_by_channel(self): |
1136 | + bpr = self.factory.makeBinaryPackageRelease( |
1137 | + binpackageformat=BinaryPackageFormat.WHL) |
1138 | + target_das = self.factory.makeDistroArchSeries() |
1139 | + args = self.makeArgs([bpr], target_das.distroseries, channel="stable") |
1140 | + [bpph] = getUtility(IPublishingSet).publishBinaries(**args) |
1141 | + self.assertContentEqual( |
1142 | + [], getUtility(IPublishingSet).publishBinaries(**args)) |
1143 | + args["channel"] = "edge" |
1144 | + [another_bpph] = getUtility(IPublishingSet).publishBinaries(**args) |
1145 | + |
1146 | |
1147 | class TestChangeOverride(TestNativePublishingBase): |
1148 | """Test that changing overrides works.""" |
1149 | |
1150 | def setUpOverride(self, status=SeriesStatus.DEVELOPMENT, |
1151 | - pocket=PackagePublishingPocket.RELEASE, binary=False, |
1152 | - ddeb=False, **kwargs): |
1153 | + pocket=PackagePublishingPocket.RELEASE, channel=None, |
1154 | + binary=False, format=None, ddeb=False, **kwargs): |
1155 | self.distroseries.status = status |
1156 | + get_pub_kwargs = {"pocket": pocket, "channel": channel} |
1157 | + if format is not None: |
1158 | + get_pub_kwargs["format"] = format |
1159 | if ddeb: |
1160 | - pub = self.getPubBinaries(pocket=pocket, with_debug=True)[2] |
1161 | + pub = self.getPubBinaries(with_debug=True, **get_pub_kwargs)[2] |
1162 | self.assertEqual( |
1163 | BinaryPackageFormat.DDEB, |
1164 | pub.binarypackagerelease.binpackageformat) |
1165 | elif binary: |
1166 | - pub = self.getPubBinaries(pocket=pocket)[0] |
1167 | + pub = self.getPubBinaries(**get_pub_kwargs)[0] |
1168 | else: |
1169 | - pub = self.getPubSource(pocket=pocket) |
1170 | + pub = self.getPubSource(**get_pub_kwargs) |
1171 | return pub.changeOverride(**kwargs) |
1172 | |
1173 | def assertCanOverride(self, status=SeriesStatus.DEVELOPMENT, |
1174 | - pocket=PackagePublishingPocket.RELEASE, **kwargs): |
1175 | - new_pub = self.setUpOverride(status=status, pocket=pocket, **kwargs) |
1176 | + pocket=PackagePublishingPocket.RELEASE, channel=None, |
1177 | + **kwargs): |
1178 | + new_pub = self.setUpOverride( |
1179 | + status=status, pocket=pocket, channel=channel, **kwargs) |
1180 | self.assertEqual(new_pub.status, PackagePublishingStatus.PENDING) |
1181 | self.assertEqual(new_pub.pocket, pocket) |
1182 | + self.assertEqual(new_pub.channel, channel) |
1183 | if "new_component" in kwargs: |
1184 | self.assertEqual(kwargs["new_component"], new_pub.component.name) |
1185 | if "new_section" in kwargs: |
1186 | @@ -1784,6 +1828,12 @@ class TestChangeOverride(TestNativePublishingBase): |
1187 | self.assertCannotOverride(new_component="partner") |
1188 | self.assertCannotOverride(binary=True, new_component="partner") |
1189 | |
1190 | + def test_preserves_channel(self): |
1191 | + self.assertCanOverride( |
1192 | + binary=True, format=BinaryPackageFormat.WHL, channel="stable", |
1193 | + new_component="universe", new_section="misc", new_priority="extra", |
1194 | + new_phased_update_percentage=90) |
1195 | + |
1196 | |
1197 | class TestPublishingHistoryView(TestCaseWithFactory): |
1198 | layer = LaunchpadFunctionalLayer |
1199 | diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py |
1200 | index 89925b6..29c6059 100644 |
1201 | --- a/lib/lp/testing/factory.py |
1202 | +++ b/lib/lp/testing/factory.py |
1203 | @@ -4029,6 +4029,7 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1204 | creator=None, |
1205 | packageupload=None, |
1206 | spr_creator=None, |
1207 | + channel=None, |
1208 | **kwargs): |
1209 | """Make a `SourcePackagePublishingHistory`. |
1210 | |
1211 | @@ -4050,6 +4051,7 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1212 | is scheduled to be removed. |
1213 | :param ancestor: The publication ancestor parameter. |
1214 | :param creator: The publication creator. |
1215 | + :param channel: An optional channel to publish into, as a string. |
1216 | :param **kwargs: All other parameters are passed through to the |
1217 | makeSourcePackageRelease call if needed. |
1218 | """ |
1219 | @@ -4087,7 +4089,7 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1220 | archive, sourcepackagerelease, distroseries, |
1221 | sourcepackagerelease.component, sourcepackagerelease.section, |
1222 | pocket, ancestor=ancestor, creator=creator, |
1223 | - packageupload=packageupload) |
1224 | + packageupload=packageupload, channel=channel) |
1225 | |
1226 | naked_spph = removeSecurityProxy(spph) |
1227 | naked_spph.status = status |
1228 | @@ -4113,7 +4115,7 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1229 | version=None, |
1230 | architecturespecific=False, |
1231 | with_debug=False, with_file=False, |
1232 | - creator=None): |
1233 | + creator=None, channel=None): |
1234 | """Make a `BinaryPackagePublishingHistory`.""" |
1235 | if distroarchseries is None: |
1236 | if archive is None: |
1237 | @@ -4144,7 +4146,10 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1238 | if priority is None: |
1239 | priority = PackagePublishingPriority.OPTIONAL |
1240 | if binpackageformat is None: |
1241 | - binpackageformat = BinaryPackageFormat.DEB |
1242 | + if binarypackagerelease is not None: |
1243 | + binpackageformat = binarypackagerelease.binpackageformat |
1244 | + else: |
1245 | + binpackageformat = BinaryPackageFormat.DEB |
1246 | |
1247 | if binarypackagerelease is None: |
1248 | # Create a new BinaryPackageBuild and BinaryPackageRelease |
1249 | @@ -4182,7 +4187,8 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1250 | archive, distroarchseries.distroseries, pocket, |
1251 | {binarypackagerelease: ( |
1252 | binarypackagerelease.component, binarypackagerelease.section, |
1253 | - priority, None)}) |
1254 | + priority, None)}, |
1255 | + channel=channel) |
1256 | for bpph in bpphs: |
1257 | naked_bpph = removeSecurityProxy(bpph) |
1258 | naked_bpph.status = status |
1259 | @@ -4256,7 +4262,7 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1260 | libraryfile=library_file, filetype=filetype)) |
1261 | |
1262 | def makeBinaryPackageRelease(self, binarypackagename=None, |
1263 | - version=None, build=None, |
1264 | + version=None, build=None, ci_build=None, |
1265 | binpackageformat=None, component=None, |
1266 | section_name=None, priority=None, |
1267 | architecturespecific=False, |
1268 | @@ -4270,22 +4276,27 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1269 | date_created=None, debug_package=None, |
1270 | homepage=None): |
1271 | """Make a `BinaryPackageRelease`.""" |
1272 | - if build is None: |
1273 | + if build is None and ci_build is None: |
1274 | build = self.makeBinaryPackageBuild() |
1275 | if binarypackagename is None or isinstance(binarypackagename, str): |
1276 | binarypackagename = self.getOrMakeBinaryPackageName( |
1277 | binarypackagename) |
1278 | - if version is None: |
1279 | + if version is None and build is not None: |
1280 | version = build.source_package_release.version |
1281 | if binpackageformat is None: |
1282 | binpackageformat = BinaryPackageFormat.DEB |
1283 | - if component is None: |
1284 | + if component is None and build is not None: |
1285 | component = build.source_package_release.component |
1286 | elif isinstance(component, str): |
1287 | component = getUtility(IComponentSet)[component] |
1288 | if isinstance(section_name, str): |
1289 | section_name = self.makeSection(section_name) |
1290 | - section = section_name or build.source_package_release.section |
1291 | + if section_name is not None: |
1292 | + section = section_name |
1293 | + elif build is not None: |
1294 | + section = build.source_package_release.section |
1295 | + else: |
1296 | + section = None |
1297 | if priority is None: |
1298 | priority = PackagePublishingPriority.OPTIONAL |
1299 | if summary is None: |
1300 | @@ -4294,18 +4305,35 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1301 | description = self.getUniqueString("description") |
1302 | if installed_size is None: |
1303 | installed_size = self.getUniqueInteger() |
1304 | - bpr = build.createBinaryPackageRelease( |
1305 | - binarypackagename=binarypackagename, version=version, |
1306 | - binpackageformat=binpackageformat, |
1307 | - component=component, section=section, priority=priority, |
1308 | - summary=summary, description=description, |
1309 | - architecturespecific=architecturespecific, |
1310 | - shlibdeps=shlibdeps, depends=depends, recommends=recommends, |
1311 | - suggests=suggests, conflicts=conflicts, replaces=replaces, |
1312 | - provides=provides, pre_depends=pre_depends, |
1313 | - enhances=enhances, breaks=breaks, essential=essential, |
1314 | - installedsize=installed_size, debug_package=debug_package, |
1315 | - homepage=homepage) |
1316 | + kwargs = { |
1317 | + "binarypackagename": binarypackagename, |
1318 | + "version": version, |
1319 | + "binpackageformat": binpackageformat, |
1320 | + "summary": summary, |
1321 | + "description": description, |
1322 | + "architecturespecific": architecturespecific, |
1323 | + "installedsize": installed_size, |
1324 | + "homepage": homepage, |
1325 | + } |
1326 | + if build is not None: |
1327 | + kwargs.update({ |
1328 | + "component": component, |
1329 | + "section": section, |
1330 | + "priority": priority, |
1331 | + "shlibdeps": shlibdeps, |
1332 | + "depends": depends, |
1333 | + "recommends": recommends, |
1334 | + "suggests": suggests, |
1335 | + "conflicts": conflicts, |
1336 | + "replaces": replaces, |
1337 | + "provides": provides, |
1338 | + "pre_depends": pre_depends, |
1339 | + "enhances": enhances, |
1340 | + "breaks": breaks, |
1341 | + "essential": essential, |
1342 | + "debug_package": debug_package, |
1343 | + }) |
1344 | + bpr = (build or ci_build).createBinaryPackageRelease(**kwargs) |
1345 | if date_created is not None: |
1346 | removeSecurityProxy(bpr).datecreated = date_created |
1347 | return bpr |