Merge ~cjwatson/launchpad:bpb-librarian-auth into launchpad:master

Proposed by Colin Watson
Status: Merged
Approved by: Colin Watson
Approved revision: 95779342f992f0c90be8fc33a26dfc2f5470888d
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: ~cjwatson/launchpad:bpb-librarian-auth
Merge into: launchpad:master
Diff against target: 223 lines (+37/-58)
2 files modified
lib/lp/soyuz/model/binarypackagebuildbehaviour.py (+8/-25)
lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py (+29/-33)
Reviewer Review Type Date Requested Status
Jürgen Gmach Approve
Review via email: mp+429703@code.launchpad.net

Commit message

Fetch files for private BPBs from librarian

Description of the change

This will allow us to lift the requirement for the source for private binary package builds to be published before we can dispatch them. We were already using macaroon authentication for private source files due to the `SnapBase` work last year; this just switches from having the private PPA server do the authorization to having the librarian do it.

We now always fetch files for binary package builds using HTTPS, even for public builds, which seems like a better idea now that we no longer need the long-fat-pipe mitigations for Boston builders.

This is roughly half of https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/373741, but rebased on master and with some more precise tests for the behaviour of public builds.

To post a comment you must log in.
Revision history for this message
Jürgen Gmach (jugmac00) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/lib/lp/soyuz/model/binarypackagebuildbehaviour.py b/lib/lp/soyuz/model/binarypackagebuildbehaviour.py
2index 79bb712..9cef29a 100644
3--- a/lib/lp/soyuz/model/binarypackagebuildbehaviour.py
4+++ b/lib/lp/soyuz/model/binarypackagebuildbehaviour.py
5@@ -22,13 +22,11 @@ from lp.buildmaster.model.buildfarmjobbehaviour import (
6 from lp.registry.interfaces.pocket import PackagePublishingPocket
7 from lp.services.config import config
8 from lp.services.twistedsupport import cancel_on_timeout
9-from lp.services.webapp import urlappend
10 from lp.soyuz.adapters.archivedependencies import (
11 get_primary_current_component,
12 get_sources_list_for_building,
13 )
14 from lp.soyuz.enums import ArchivePurpose
15-from lp.soyuz.model.publishing import makePoolPath
16
17
18 @implementer(IBuildFarmJobBehaviour)
19@@ -70,35 +68,20 @@ class BinaryPackageBuildBehaviour(BuildFarmJobBehaviourBase):
20 """See `IBuildFarmJobBehaviour`."""
21 # Build filemap structure with the files required in this build
22 # and send them to the worker.
23- if self.build.archive.private:
24- # Builds in private archive may have restricted files that
25- # we can't obtain from the public librarian. Prepare a pool
26- # URL from which to fetch them.
27- pool_url = urlappend(
28- self.build.archive.archive_url,
29- makePoolPath(
30- self.build.source_package_release.sourcepackagename.name,
31- self.build.current_component.name,
32- ),
33- )
34 filemap = OrderedDict()
35 macaroon_raw = None
36 for source_file in self.build.source_package_release.files:
37 lfa = source_file.libraryfile
38- if not self.build.archive.private:
39- filemap[lfa.filename] = {
40- "sha1": lfa.content.sha1,
41- "url": lfa.http_url,
42- }
43- else:
44+ filemap[lfa.filename] = {
45+ "sha1": lfa.content.sha1,
46+ "url": lfa.https_url,
47+ }
48+ if self.build.archive.private:
49 if macaroon_raw is None:
50 macaroon_raw = yield self.issueMacaroon()
51- filemap[lfa.filename] = {
52- "sha1": lfa.content.sha1,
53- "url": urlappend(pool_url, lfa.filename),
54- "username": "buildd",
55- "password": macaroon_raw,
56- }
57+ filemap[lfa.filename].update(
58+ username="", password=macaroon_raw
59+ )
60 return filemap
61
62 def verifyBuildRequest(self, logger):
63diff --git a/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py b/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py
64index 2d1a99b..f3625ec 100644
65--- a/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py
66+++ b/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py
67@@ -18,7 +18,6 @@ from twisted.internet import defer
68 from zope.component import getUtility
69 from zope.security.proxy import removeSecurityProxy
70
71-from lp.archivepublisher.diskpool import poolify
72 from lp.archivepublisher.interfaces.archivegpgsigningkey import (
73 IArchiveGPGSigningKey,
74 )
75@@ -92,8 +91,6 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
76 archive,
77 archive_purpose,
78 component=None,
79- extra_uploads=None,
80- filemap_names=None,
81 ):
82 matcher = yield self.makeExpectedInteraction(
83 builder,
84@@ -103,8 +100,6 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
85 archive,
86 archive_purpose,
87 component,
88- extra_uploads,
89- filemap_names,
90 )
91 self.assertThat(call_log, matcher)
92
93@@ -118,8 +113,6 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
94 archive,
95 archive_purpose,
96 component=None,
97- extra_uploads=None,
98- filemap_names=None,
99 ):
100 """Build the log of calls that we expect to be made to the worker.
101
102@@ -144,11 +137,17 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
103 arch_indep = das.isNominatedArchIndep
104 if component is None:
105 component = build.current_component.name
106- if filemap_names is None:
107- filemap_names = []
108- if extra_uploads is None:
109- extra_uploads = []
110+ files = build.source_package_release.files
111
112+ uploads = [(chroot.http_url, "", "")]
113+ for sprf in files:
114+ if build.archive.private:
115+ password = MacaroonVerifies(
116+ "binary-package-build", build.archive
117+ )
118+ else:
119+ password = ""
120+ uploads.append((sprf.libraryfile.https_url, "", password))
121 upload_logs = [
122 MatchesListwise(
123 [Equals("ensurepresent")]
124@@ -157,7 +156,7 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
125 for item in upload
126 ]
127 )
128- for upload in [(chroot.http_url, "", "")] + extra_uploads
129+ for upload in uploads
130 ]
131
132 extra_args = {
133@@ -182,7 +181,7 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
134 build.build_cookie,
135 "binarypackage",
136 chroot.content.sha1,
137- filemap_names,
138+ [sprf.libraryfile.filename for sprf in files],
139 extra_args,
140 )
141 ]
142@@ -208,6 +207,10 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
143 build = self.factory.makeBinaryPackageBuild(
144 builder=builder, archive=archive
145 )
146+ build.source_package_release.addFile(
147+ self.factory.makeLibraryFileAlias(db_only=True),
148+ filetype=SourcePackageFileType.ORIG_TARBALL,
149+ )
150 lf = self.factory.makeLibraryFileAlias(db_only=True)
151 build.distro_arch_series.addOrUpdateChroot(lf)
152 bq = build.queueBuild()
153@@ -248,6 +251,10 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
154 build = self.factory.makeBinaryPackageBuild(
155 builder=builder, archive=archive
156 )
157+ build.source_package_release.addFile(
158+ self.factory.makeLibraryFileAlias(db_only=True),
159+ filetype=SourcePackageFileType.ORIG_TARBALL,
160+ )
161 self.factory.makeSourcePackagePublishingHistory(
162 distroseries=build.distro_series,
163 archive=archive.distribution.main_archive,
164@@ -284,6 +291,10 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
165 build = self.factory.makeBinaryPackageBuild(
166 builder=builder, archive=archive
167 )
168+ build.source_package_release.addFile(
169+ self.factory.makeLibraryFileAlias(db_only=True),
170+ filetype=SourcePackageFileType.ORIG_TARBALL,
171+ )
172 lf = self.factory.makeLibraryFileAlias(db_only=True)
173 build.distro_arch_series.addOrUpdateChroot(lf)
174 bq = build.queueBuild()
175@@ -325,21 +336,10 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
176 build = self.factory.makeBinaryPackageBuild(
177 builder=builder, archive=archive
178 )
179- sprf = build.source_package_release.addFile(
180+ build.source_package_release.addFile(
181 self.factory.makeLibraryFileAlias(db_only=True),
182 filetype=SourcePackageFileType.ORIG_TARBALL,
183 )
184- sprf_url = (
185- "http://private-ppa.launchpad.test/%s/%s/ubuntu/pool/%s/%s"
186- % (
187- archive.owner.name,
188- archive.name,
189- poolify(
190- build.source_package_release.sourcepackagename.name, "main"
191- ).as_posix(),
192- sprf.libraryfile.filename,
193- )
194- )
195 lf = self.factory.makeLibraryFileAlias(db_only=True)
196 build.distro_arch_series.addOrUpdateChroot(lf)
197 bq = build.queueBuild()
198@@ -357,14 +357,6 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
199 lf,
200 archive,
201 ArchivePurpose.PPA,
202- extra_uploads=[
203- (
204- Equals(sprf_url),
205- Equals("buildd"),
206- MacaroonVerifies("binary-package-build", archive),
207- )
208- ],
209- filemap_names=[sprf.libraryfile.filename],
210 )
211
212 @defer.inlineCallbacks
213@@ -379,6 +371,10 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
214 build = self.factory.makeBinaryPackageBuild(
215 builder=builder, archive=archive
216 )
217+ build.source_package_release.addFile(
218+ self.factory.makeLibraryFileAlias(db_only=True),
219+ filetype=SourcePackageFileType.ORIG_TARBALL,
220+ )
221 lf = self.factory.makeLibraryFileAlias(db_only=True)
222 build.distro_arch_series.addOrUpdateChroot(lf)
223 bq = build.queueBuild()

Subscribers

People subscribed via source and target branches

to status/vote changes: