Merge lp:~cjwatson/launchpad/build-private-bpb-immediately into lp:launchpad

Proposed by Colin Watson on 2018-05-04
Status: Needs review
Proposed branch: lp:~cjwatson/launchpad/build-private-bpb-immediately
Merge into: lp:launchpad
Prerequisite: lp:~cjwatson/launchpad/librarian-accept-macaroon
Diff against target: 276 lines (+60/-64)
4 files modified
lib/lp/buildmaster/tests/test_builder.py (+4/-4)
lib/lp/soyuz/model/binarypackagebuild.py (+3/-27)
lib/lp/soyuz/model/binarypackagebuildbehaviour.py (+12/-17)
lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py (+41/-16)
To merge this branch: bzr merge lp:~cjwatson/launchpad/build-private-bpb-immediately
Reviewer Review Type Date Requested Status
Launchpad code reviewers 2018-05-04 Pending
Review via email: mp+345104@code.launchpad.net

Commit message

Dispatch private BPBs immediately, using macaroon auth for source files.

Description of the change

The prerequisite branch must be deployed to the librarian before landing this.

I haven't yet tested this beyond what's in the test suite.

To post a comment you must log in.

Unmerged revisions

18632. By Colin Watson on 2018-05-04

Dispatch private BPBs immediately, using macaroon auth for source files.

18631. By Colin Watson on 2018-05-04

Allow macaroon authentication to the librarian for BPBs' source files.

This will later allow us to dispatch private builds immediately, rather than
having to wait for the source to be published first.

In this case we accept the macaroon as the password part of basic
authentication with an empty username, similar to the git authentication
method used by code import jobs. This is a bit weird, but it avoids needing
to change launchpad-buildd, and we could always switch to a more
conventional "Authorization: Macaroon ..." scheme later.

18630. By Colin Watson on 2018-05-04

Make CodeImportJobMacaroonIssuer.verifyMacaroon check that the context is the right type, just in case.

18629. By Colin Watson on 2018-05-04

Convert lp.services.librarianserver.tests.test_web to requests.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/buildmaster/tests/test_builder.py'
2--- lib/lp/buildmaster/tests/test_builder.py 2018-01-10 10:45:24 +0000
3+++ lib/lp/buildmaster/tests/test_builder.py 2018-05-04 17:00:50 +0000
4@@ -401,13 +401,13 @@
5 build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job)
6 self.assertEqual('joesppa', build.archive.name)
7
8- # If the source for the build is still pending, it won't be
9- # dispatched because the builder has to fetch the source files
10- # from the (password protected) repo area, not the librarian.
11+ # If the source for the build is still pending, it will still be
12+ # dispatched: the builder will use macaroon authentication to fetch
13+ # the source files from the librarian.
14 pub = build.current_source_publication
15 pub.status = PackagePublishingStatus.PENDING
16 candidate = removeSecurityProxy(self.builder4)._findBuildCandidate()
17- self.assertNotEqual(next_job.id, candidate.id)
18+ self.assertEqual(next_job.id, candidate.id)
19
20
21 class TestFindBuildCandidateDistroArchive(TestFindBuildCandidateBase):
22
23=== modified file 'lib/lp/soyuz/model/binarypackagebuild.py'
24--- lib/lp/soyuz/model/binarypackagebuild.py 2018-05-04 17:00:50 +0000
25+++ lib/lp/soyuz/model/binarypackagebuild.py 2018-05-04 17:00:50 +0000
26@@ -87,10 +87,7 @@
27 )
28 from lp.services.macaroons.interfaces import IMacaroonIssuer
29 from lp.soyuz.adapters.buildarch import determine_architectures_to_build
30-from lp.soyuz.enums import (
31- ArchivePurpose,
32- PackagePublishingStatus,
33- )
34+from lp.soyuz.enums import ArchivePurpose
35 from lp.soyuz.interfaces.archive import (
36 InvalidExternalDependencies,
37 validate_external_dependencies,
38@@ -1210,33 +1207,12 @@
39 @staticmethod
40 def addCandidateSelectionCriteria(processor, virtualized):
41 """See `ISpecificBuildFarmJobSource`."""
42- private_statuses = (
43- PackagePublishingStatus.PUBLISHED,
44- PackagePublishingStatus.SUPERSEDED,
45- PackagePublishingStatus.DELETED,
46- )
47 return """
48- SELECT TRUE FROM Archive, BinaryPackageBuild, DistroArchSeries
49+ SELECT TRUE FROM BinaryPackageBuild
50 WHERE
51 BinaryPackageBuild.build_farm_job = BuildQueue.build_farm_job AND
52- BinaryPackageBuild.distro_arch_series =
53- DistroArchSeries.id AND
54- BinaryPackageBuild.archive = Archive.id AND
55- ((Archive.private IS TRUE AND
56- EXISTS (
57- SELECT SourcePackagePublishingHistory.id
58- FROM SourcePackagePublishingHistory
59- WHERE
60- SourcePackagePublishingHistory.distroseries =
61- DistroArchSeries.distroseries AND
62- SourcePackagePublishingHistory.sourcepackagerelease =
63- BinaryPackageBuild.source_package_release AND
64- SourcePackagePublishingHistory.archive = Archive.id AND
65- SourcePackagePublishingHistory.status IN %s))
66- OR
67- archive.private IS FALSE) AND
68 BinaryPackageBuild.status = %s
69- """ % sqlvalues(private_statuses, BuildStatus.NEEDSBUILD)
70+ """ % sqlvalues(BuildStatus.NEEDSBUILD)
71
72 @staticmethod
73 def postprocessCandidate(job, logger):
74
75=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehaviour.py'
76--- lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2018-03-01 17:36:31 +0000
77+++ lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2018-05-04 17:00:50 +0000
78@@ -1,4 +1,4 @@
79-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
80+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
81 # GNU Affero General Public License version 3 (see the file LICENSE).
82
83 """Builder behaviour for binary package builds."""
84@@ -10,7 +10,9 @@
85 ]
86
87 from twisted.internet import defer
88+from zope.component import getUtility
89 from zope.interface import implementer
90+from zope.security.proxy import removeSecurityProxy
91
92 from lp.buildmaster.interfaces.builder import CannotBuild
93 from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
94@@ -20,16 +22,13 @@
95 BuildFarmJobBehaviourBase,
96 )
97 from lp.registry.interfaces.pocket import PackagePublishingPocket
98-from lp.services.webapp import (
99- canonical_url,
100- urlappend,
101- )
102+from lp.services.macaroons.interfaces import IMacaroonIssuer
103+from lp.services.webapp import canonical_url
104 from lp.soyuz.adapters.archivedependencies import (
105 get_primary_current_component,
106 get_sources_list_for_building,
107 )
108 from lp.soyuz.enums import ArchivePurpose
109-from lp.soyuz.model.publishing import makePoolPath
110
111
112 @implementer(IBuildFarmJobBehaviour)
113@@ -63,14 +62,9 @@
114 # Build filemap structure with the files required in this build
115 # and send them to the slave.
116 if self.build.archive.private:
117- # Builds in private archive may have restricted files that
118- # we can't obtain from the public librarian. Prepare a pool
119- # URL from which to fetch them.
120- pool_url = urlappend(
121- self.build.archive.archive_url,
122- makePoolPath(
123- self.build.source_package_release.sourcepackagename.name,
124- self.build.current_component.name))
125+ issuer = getUtility(IMacaroonIssuer, 'binary-package-build')
126+ password = removeSecurityProxy(issuer).issueMacaroon(
127+ self.build).serialize()
128 filemap = {}
129 for source_file in self.build.source_package_release.files:
130 lfa = source_file.libraryfile
131@@ -80,9 +74,10 @@
132 else:
133 filemap[lfa.filename] = {
134 'sha1': lfa.content.sha1,
135- 'url': urlappend(pool_url, lfa.filename),
136- 'username': 'buildd',
137- 'password': self.build.archive.buildd_secret}
138+ 'url': lfa.https_url,
139+ 'username': '',
140+ 'password': password,
141+ }
142 return filemap
143
144 @defer.inlineCallbacks
145
146=== modified file 'lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py'
147--- lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2018-03-01 17:36:31 +0000
148+++ lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2018-05-04 17:00:50 +0000
149@@ -12,8 +12,15 @@
150 import shutil
151 import tempfile
152
153+from pymacaroons import Macaroon
154 from storm.store import Store
155-from testtools.matchers import MatchesListwise
156+from testtools.matchers import (
157+ Contains,
158+ Equals,
159+ Matcher,
160+ MatchesListwise,
161+ Mismatch,
162+ )
163 from testtools.twistedsupport import AsynchronousDeferredRunTest
164 import transaction
165 from twisted.internet import defer
166@@ -21,7 +28,6 @@
167 from zope.component import getUtility
168 from zope.security.proxy import removeSecurityProxy
169
170-from lp.archivepublisher.diskpool import poolify
171 from lp.archivepublisher.interfaces.archivesigningkey import (
172 IArchiveSigningKey,
173 )
174@@ -58,6 +64,7 @@
175 from lp.services.config import config
176 from lp.services.librarian.interfaces import ILibraryFileAliasSet
177 from lp.services.log.logger import BufferLogger
178+from lp.services.macaroons.interfaces import IMacaroonIssuer
179 from lp.services.webapp import canonical_url
180 from lp.soyuz.adapters.archivedependencies import (
181 get_sources_list_for_building,
182@@ -74,6 +81,26 @@
183 from lp.testing.layers import LaunchpadZopelessLayer
184
185
186+class BinaryPackageBuildMacaroonVerifies(Matcher):
187+ """Matches if a binary-package-build macaroon passes verification."""
188+
189+ def __init__(self, build, lfa_id):
190+ self.build = build
191+ self.lfa_id = lfa_id
192+
193+ def match(self, macaroon_raw):
194+ macaroon = Macaroon.deserialize(macaroon_raw)
195+ expected_caveat = (
196+ "lp.binary-package-build %s" % removeSecurityProxy(self.build).id)
197+ mismatch = Contains(expected_caveat).match(
198+ [caveat.caveat_id for caveat in macaroon.caveats])
199+ if mismatch is not None:
200+ return mismatch
201+ issuer = getUtility(IMacaroonIssuer, "binary-package-build")
202+ if not issuer.verifyMacaroon(macaroon, self.lfa_id):
203+ return Mismatch("Macaroon does not verify")
204+
205+
206 class TestBinaryBuildPackageBehaviour(TestCaseWithFactory):
207 """Tests for the BinaryPackageBuildBehaviour.
208
209@@ -98,7 +125,7 @@
210 expected = yield self.makeExpectedInteraction(
211 builder, build, chroot, archive, archive_purpose, component,
212 extra_uploads, filemap_names)
213- self.assertEqual(expected, call_log)
214+ self.assertThat(call_log, MatchesListwise(expected))
215
216 @defer.inlineCallbacks
217 def makeExpectedInteraction(self, builder, build, chroot, archive,
218@@ -115,7 +142,7 @@
219 builder. We specify this separately from the archive because
220 sometimes the behaviour object has to give a different purpose
221 in order to trick the slave into building correctly.
222- :return: A list of the calls we expect to be made.
223+ :return: A list of matchers for the calls we expect to be made.
224 """
225 das = build.distro_arch_series
226 ds_name = das.distroseries.name
227@@ -131,8 +158,9 @@
228 extra_uploads = []
229
230 upload_logs = [
231- ('ensurepresent',) + upload
232- for upload in [(chroot.http_url, '', '')] + extra_uploads]
233+ MatchesListwise((Equals('ensurepresent'),) + upload)
234+ for upload in [(Equals(chroot.http_url), Equals(''), Equals(''))] +
235+ extra_uploads]
236
237 extra_args = {
238 'arch_indep': arch_indep,
239@@ -149,8 +177,9 @@
240 'trusted_keys': trusted_keys,
241 }
242 build_log = [
243- ('build', build.build_cookie, 'binarypackage',
244- chroot.content.sha1, filemap_names, extra_args)]
245+ MatchesListwise([Equals(arg) for arg in (
246+ 'build', build.build_cookie, 'binarypackage',
247+ chroot.content.sha1, filemap_names, extra_args)])]
248 result = upload_logs + build_log
249 defer.returnValue(result)
250
251@@ -244,13 +273,6 @@
252 sprf = build.source_package_release.addFile(
253 self.factory.makeLibraryFileAlias(db_only=True),
254 filetype=SourcePackageFileType.ORIG_TARBALL)
255- sprf_url = (
256- 'http://private-ppa.launchpad.dev/%s/%s/ubuntu/pool/%s/%s'
257- % (archive.owner.name, archive.name,
258- poolify(
259- build.source_package_release.sourcepackagename.name,
260- 'main'),
261- sprf.libraryfile.filename))
262 lf = self.factory.makeLibraryFileAlias()
263 transaction.commit()
264 build.distro_arch_series.addOrUpdateChroot(lf)
265@@ -262,7 +284,10 @@
266 interactor.getBuildBehaviour(bq, builder, slave), BufferLogger())
267 yield self.assertExpectedInteraction(
268 slave.call_log, builder, build, lf, archive, ArchivePurpose.PPA,
269- extra_uploads=[(sprf_url, 'buildd', 'sekrit')],
270+ extra_uploads=[(
271+ Equals(sprf.libraryfile.https_url), Equals(''),
272+ BinaryPackageBuildMacaroonVerifies(
273+ build, sprf.libraryfile.id))],
274 filemap_names=[sprf.libraryfile.filename])
275
276 @defer.inlineCallbacks