Launchpad itself

Merge lp:~cjwatson/launchpad/send-keys-to-builders into lp:launchpad

  1. send-keys-to-builders
  2. Merge into devel
Proposed by Colin Watson
Status: Merged
Merged at revision: 18405
Proposed branch: lp:~cjwatson/launchpad/send-keys-to-builders
Merge into: lp:launchpad
Prerequisite: lp:~cjwatson/launchpad/faster-archive-signing-key-tests
Diff against target: 1502 lines (+445/-123)
14 files modified
lib/lp/code/model/recipebuilder.py (+5/-4)
lib/lp/code/model/tests/test_recipebuilder.py (+51/-11)
lib/lp/services/tests/test_timeout.py (+10/-0)
lib/lp/services/timeout.py (+18/-0)
lib/lp/snappy/model/snapbuildbehaviour.py (+4/-3)
lib/lp/snappy/tests/test_snapbuildbehaviour.py (+59/-16)
lib/lp/soyuz/adapters/archivedependencies.py (+54/-13)
lib/lp/soyuz/adapters/tests/test_archivedependencies.py (+129/-61)
lib/lp/soyuz/model/binarypackagebuildbehaviour.py (+3/-2)
lib/lp/soyuz/model/livefsbuildbehaviour.py (+3/-2)
lib/lp/soyuz/tests/soyuz.py (+21/-1)
lib/lp/soyuz/tests/test_archive.py (+20/-3)
lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py (+30/-2)
lib/lp/soyuz/tests/test_livefsbuildbehaviour.py (+38/-5)
To merge this branch: bzr merge lp:~cjwatson/launchpad/send-keys-to-builders
Reviewer Review Type Date Requested Status
William Grant code Approve
Review via email: mp+323438@code.launchpad.net

Commit message

Send the necessary set of archive signing keys to builders.

To post a comment you must log in.
Revision history for this message
William Grant (wgrant) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/code/model/recipebuilder.py'
2--- lib/lp/code/model/recipebuilder.py 2017-04-28 17:30:55 +0000
3+++ lib/lp/code/model/recipebuilder.py 2017-06-13 17:24:41 +0000
4@@ -67,10 +67,11 @@
5 args["ogrecomponent"] = get_primary_current_component(
6 self.build.archive, self.build.distroseries,
7 None).name
8- args['archives'] = yield get_sources_list_for_building(
9- self.build, distroarchseries, None,
10- tools_source=config.builddmaster.bzr_builder_sources_list,
11- logger=logger)
12+ args['archives'], args['trusted_keys'] = (
13+ yield get_sources_list_for_building(
14+ self.build, distroarchseries, None,
15+ tools_source=config.builddmaster.bzr_builder_sources_list,
16+ logger=logger))
17 args['archive_private'] = self.build.archive.private
18 args['distroseries_name'] = self.build.distroseries.name
19 if self.build.recipe.base_git_repository is not None:
20
21=== modified file 'lib/lp/code/model/tests/test_recipebuilder.py'
22--- lib/lp/code/model/tests/test_recipebuilder.py 2017-04-28 17:30:55 +0000
23+++ lib/lp/code/model/tests/test_recipebuilder.py 2017-06-13 17:24:41 +0000
24@@ -5,16 +5,21 @@
25
26 __metaclass__ = type
27
28+import os.path
29 import shutil
30 import tempfile
31
32 from testtools.deferredruntest import AsynchronousDeferredRunTest
33+from testtools.matchers import MatchesListwise
34 import transaction
35 from twisted.internet import defer
36 from twisted.trial.unittest import TestCase as TrialTestCase
37 from zope.component import getUtility
38 from zope.security.proxy import removeSecurityProxy
39
40+from lp.archivepublisher.interfaces.archivesigningkey import (
41+ IArchiveSigningKey,
42+ )
43 from lp.buildmaster.enums import BuildStatus
44 from lp.buildmaster.interactor import BuilderInteractor
45 from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
46@@ -40,13 +45,19 @@
47 from lp.soyuz.adapters.archivedependencies import (
48 get_sources_list_for_building,
49 )
50-from lp.soyuz.enums import ArchivePurpose
51+from lp.soyuz.enums import (
52+ ArchivePurpose,
53+ PackagePublishingStatus,
54+ )
55+from lp.soyuz.tests.soyuz import Base64KeyMatches
56 from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
57 from lp.testing import (
58 person_logged_in,
59 TestCaseWithFactory,
60 )
61 from lp.testing.fakemethod import FakeMethod
62+from lp.testing.gpgkeys import gpgkeysdir
63+from lp.testing.keyserver import InProcessKeyServerFixture
64 from lp.testing.layers import LaunchpadZopelessLayer
65 from lp.testing.mail_helpers import pop_notifications
66
67@@ -59,7 +70,10 @@
68 archive=None, git=False):
69 """Create a sample `ISourcePackageRecipeBuild`."""
70 spn = self.factory.makeSourcePackageName("apackage")
71- distro = self.factory.makeDistribution(name="distro")
72+ if archive is None:
73+ distro = self.factory.makeDistribution(name="distro")
74+ else:
75+ distro = archive.distribution
76 distroseries = self.factory.makeDistroSeries(
77 name="mydistro", distribution=distro)
78 processor = getUtility(IProcessorSet).getByName('386')
79@@ -144,7 +158,7 @@
80
81 class TestAsyncRecipeBuilder(TestRecipeBuilderBase):
82
83- run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=30)
84+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=10)
85
86 def _setBuilderConfig(self):
87 """Setup a temporary builder config."""
88@@ -158,8 +172,9 @@
89 self._setBuilderConfig()
90 job = self.makeJob()
91 distroarchseries = job.build.distroseries.architectures[0]
92- expected_archives = yield get_sources_list_for_building(
93- job.build, distroarchseries, None)
94+ expected_archives, expected_trusted_keys = (
95+ yield get_sources_list_for_building(
96+ job.build, distroarchseries, None))
97 expected_archives.insert(
98 0, "deb http://foo %s main" % job.build.distroseries.name)
99 args = yield job._extraBuildArgs(distroarchseries)
100@@ -175,6 +190,7 @@
101 '# bzr-builder format 0.3 deb-version {debupstream}-0~{revno}\n'
102 'lp://dev/~joe/someapp/pkg\n',
103 'archives': expected_archives,
104+ 'trusted_keys': expected_trusted_keys,
105 'distroseries_name': job.build.distroseries.name,
106 }, args)
107
108@@ -249,8 +265,9 @@
109 # (note the missing 's' in %(series)
110 job = self.makeJob()
111 distroarchseries = job.build.distroseries.architectures[0]
112- expected_archives = yield get_sources_list_for_building(
113- job.build, distroarchseries, None)
114+ expected_archives, expected_trusted_keys = (
115+ yield get_sources_list_for_building(
116+ job.build, distroarchseries, None))
117 logger = BufferLogger()
118 extra_args = yield job._extraBuildArgs(distroarchseries, logger)
119 self.assertEqual({
120@@ -265,6 +282,7 @@
121 '# bzr-builder format 0.3 deb-version {debupstream}-0~{revno}\n'
122 'lp://dev/~joe/someapp/pkg\n',
123 'archives': expected_archives,
124+ 'trusted_keys': expected_trusted_keys,
125 'distroseries_name': job.build.distroseries.name,
126 }, extra_args)
127 self.assertIn(
128@@ -278,16 +296,19 @@
129 job = self.makeJob()
130 distroarchseries = job.build.distroseries.architectures[0]
131 args = yield job._extraBuildArgs(distroarchseries)
132- expected_archives = yield get_sources_list_for_building(
133- job.build, distroarchseries, None)
134+ expected_archives, expected_trusted_keys = (
135+ yield get_sources_list_for_building(
136+ job.build, distroarchseries, None))
137 self.assertEqual(args["archives"], expected_archives)
138+ self.assertEqual(args["trusted_keys"], expected_trusted_keys)
139
140 @defer.inlineCallbacks
141 def test_extraBuildArgs_git(self):
142 job = self.makeJob(git=True)
143 distroarchseries = job.build.distroseries.architectures[0]
144- expected_archives = yield get_sources_list_for_building(
145- job.build, distroarchseries, None)
146+ expected_archives, expected_trusted_keys = (
147+ yield get_sources_list_for_building(
148+ job.build, distroarchseries, None))
149 extra_args = yield job._extraBuildArgs(distroarchseries)
150 self.assertEqual({
151 'archive_private': False,
152@@ -302,11 +323,30 @@
153 '{debupstream}-0~{revtime}\n'
154 'lp:~joe/someapp/+git/pkg packaging\n',
155 'archives': expected_archives,
156+ 'trusted_keys': expected_trusted_keys,
157 'distroseries_name': job.build.distroseries.name,
158 'git': True,
159 }, extra_args)
160
161 @defer.inlineCallbacks
162+ def test_extraBuildArgs_archive_trusted_keys(self):
163+ # If the archive has a signing key, _extraBuildArgs sends it.
164+ yield self.useFixture(InProcessKeyServerFixture()).start()
165+ archive = self.factory.makeArchive()
166+ key_path = os.path.join(gpgkeysdir, "ppa-sample@canonical.com.sec")
167+ yield IArchiveSigningKey(archive).setSigningKey(
168+ key_path, async_keyserver=True)
169+ job = self.makeJob(archive=archive)
170+ distroarchseries = job.build.distroseries.architectures[0]
171+ self.factory.makeBinaryPackagePublishingHistory(
172+ distroarchseries=distroarchseries, pocket=job.build.pocket,
173+ archive=archive, status=PackagePublishingStatus.PUBLISHED)
174+ args = yield job._extraBuildArgs(distroarchseries)
175+ self.assertThat(args["trusted_keys"], MatchesListwise([
176+ Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"),
177+ ]))
178+
179+ @defer.inlineCallbacks
180 def test_composeBuildRequest(self):
181 job = self.makeJob()
182 test_publisher = SoyuzTestPublisher()
183
184=== modified file 'lib/lp/services/tests/test_timeout.py'
185--- lib/lp/services/tests/test_timeout.py 2016-07-01 20:07:09 +0000
186+++ lib/lp/services/tests/test_timeout.py 2017-06-13 17:24:41 +0000
187@@ -22,6 +22,7 @@
188 from testtools.matchers import MatchesStructure
189
190 from lp.services.timeout import (
191+ default_timeout,
192 get_default_timeout_function,
193 reduced_timeout,
194 set_default_timeout_function,
195@@ -207,6 +208,15 @@
196 no_default_timeout()
197 self.assertEqual([True], using_default)
198
199+ def test_default_timeout(self):
200+ """default_timeout sets the default timeout if none is set."""
201+ self.addCleanup(set_default_timeout_function, None)
202+ with default_timeout(1.0):
203+ self.assertEqual(1.0, get_default_timeout_function()())
204+ set_default_timeout_function(lambda: 5.0)
205+ with default_timeout(1.0):
206+ self.assertEqual(5.0, get_default_timeout_function()())
207+
208 def test_reduced_timeout(self):
209 """reduced_timeout caps the available timeout in various ways."""
210 self.addCleanup(set_default_timeout_function, None)
211
212=== modified file 'lib/lp/services/timeout.py'
213--- lib/lp/services/timeout.py 2017-05-05 11:48:31 +0000
214+++ lib/lp/services/timeout.py 2017-06-13 17:24:41 +0000
215@@ -5,6 +5,7 @@
216
217 __metaclass__ = type
218 __all__ = [
219+ "default_timeout",
220 "get_default_timeout_function",
221 "reduced_timeout",
222 "SafeTransportWithTimeout",
223@@ -57,6 +58,23 @@
224
225
226 @contextmanager
227+def default_timeout(default):
228+ """A context manager that sets the default timeout if none is set.
229+
230+ :param default: The default timeout to use if none is set.
231+ """
232+ original_timeout_function = get_default_timeout_function()
233+
234+ if original_timeout_function is None:
235+ set_default_timeout_function(lambda: default)
236+ try:
237+ yield
238+ finally:
239+ if original_timeout_function is None:
240+ set_default_timeout_function(None)
241+
242+
243+@contextmanager
244 def reduced_timeout(clearance, webapp_max=None, default=None):
245 """A context manager that reduces the default timeout.
246
247
248=== modified file 'lib/lp/snappy/model/snapbuildbehaviour.py'
249--- lib/lp/snappy/model/snapbuildbehaviour.py 2017-05-15 10:12:40 +0000
250+++ lib/lp/snappy/model/snapbuildbehaviour.py 2017-06-13 17:24:41 +0000
251@@ -99,9 +99,10 @@
252 args["arch_tag"] = build.distro_arch_series.architecturetag
253 # XXX cjwatson 2015-08-03: Allow tools_source to be overridden at
254 # some more fine-grained level.
255- args["archives"] = yield get_sources_list_for_building(
256- build, build.distro_arch_series, None,
257- tools_source=config.snappy.tools_source, logger=logger)
258+ args["archives"], args["trusted_keys"] = (
259+ yield get_sources_list_for_building(
260+ build, build.distro_arch_series, None,
261+ tools_source=config.snappy.tools_source, logger=logger))
262 args["archive_private"] = build.archive.private
263 if build.snap.branch is not None:
264 args["branch"] = build.snap.branch.bzr_identity
265
266=== modified file 'lib/lp/snappy/tests/test_snapbuildbehaviour.py'
267--- lib/lp/snappy/tests/test_snapbuildbehaviour.py 2017-05-15 10:12:40 +0000
268+++ lib/lp/snappy/tests/test_snapbuildbehaviour.py 2017-06-13 17:24:41 +0000
269@@ -6,6 +6,7 @@
270 __metaclass__ = type
271
272 from datetime import datetime
273+import os.path
274 from textwrap import dedent
275 import uuid
276
277@@ -17,13 +18,19 @@
278 from pymacaroons import Macaroon
279 from testtools import ExpectedException
280 from testtools.deferredruntest import AsynchronousDeferredRunTest
281-from testtools.matchers import IsInstance
282+from testtools.matchers import (
283+ IsInstance,
284+ MatchesListwise,
285+ )
286 import transaction
287 from twisted.internet import defer
288 from twisted.trial.unittest import TestCase as TrialTestCase
289 from zope.component import getUtility
290 from zope.security.proxy import removeSecurityProxy
291
292+from lp.archivepublisher.interfaces.archivesigningkey import (
293+ IArchiveSigningKey,
294+ )
295 from lp.buildmaster.enums import BuildStatus
296 from lp.buildmaster.interfaces.builder import CannotBuild
297 from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
298@@ -48,8 +55,12 @@
299 from lp.soyuz.adapters.archivedependencies import (
300 get_sources_list_for_building,
301 )
302+from lp.soyuz.enums import PackagePublishingStatus
303 from lp.soyuz.interfaces.archive import ArchiveDisabled
304+from lp.soyuz.tests.soyuz import Base64KeyMatches
305 from lp.testing import TestCaseWithFactory
306+from lp.testing.gpgkeys import gpgkeysdir
307+from lp.testing.keyserver import InProcessKeyServerFixture
308 from lp.testing.layers import LaunchpadZopelessLayer
309
310
311@@ -60,9 +71,13 @@
312 super(TestSnapBuildBehaviourBase, self).setUp()
313 self.pushConfig("snappy", tools_source=None)
314
315- def makeJob(self, pocket=PackagePublishingPocket.UPDATES, **kwargs):
316+ def makeJob(self, archive=None, pocket=PackagePublishingPocket.UPDATES,
317+ **kwargs):
318 """Create a sample `ISnapBuildBehaviour`."""
319- distribution = self.factory.makeDistribution(name="distro")
320+ if archive is None:
321+ distribution = self.factory.makeDistribution(name="distro")
322+ else:
323+ distribution = archive.distribution
324 distroseries = self.factory.makeDistroSeries(
325 distribution=distribution, name="unstable")
326 processor = getUtility(IProcessorSet).getByName("386")
327@@ -70,7 +85,7 @@
328 distroseries=distroseries, architecturetag="i386",
329 processor=processor)
330 build = self.factory.makeSnapBuild(
331- distroarchseries=distroarchseries, pocket=pocket,
332+ archive=archive, distroarchseries=distroarchseries, pocket=pocket,
333 name=u"test-snap", **kwargs)
334 return IBuildFarmJobBehaviour(build)
335
336@@ -169,7 +184,7 @@
337
338
339 class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
340- run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=30)
341+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=10)
342
343 def setUp(self):
344 super(TestAsyncSnapBuildBehaviour, self).setUp()
345@@ -212,8 +227,9 @@
346 # job for a Bazaar branch.
347 branch = self.factory.makeBranch()
348 job = self.makeJob(branch=branch)
349- expected_archives = yield get_sources_list_for_building(
350- job.build, job.build.distro_arch_series, None)
351+ expected_archives, expected_trusted_keys = (
352+ yield get_sources_list_for_building(
353+ job.build, job.build.distro_arch_series, None))
354 args = yield job._extraBuildArgs()
355 self.assertEqual({
356 "archive_private": False,
357@@ -223,6 +239,7 @@
358 "name": u"test-snap",
359 "proxy_url": self.proxy_url,
360 "revocation_endpoint": self.revocation_endpoint,
361+ "trusted_keys": expected_trusted_keys,
362 }, args)
363
364 @defer.inlineCallbacks
365@@ -231,8 +248,9 @@
366 # job for a Git branch.
367 [ref] = self.factory.makeGitRefs()
368 job = self.makeJob(git_ref=ref)
369- expected_archives = yield get_sources_list_for_building(
370- job.build, job.build.distro_arch_series, None)
371+ expected_archives, expected_trusted_keys = (
372+ yield get_sources_list_for_building(
373+ job.build, job.build.distro_arch_series, None))
374 args = yield job._extraBuildArgs()
375 self.assertEqual({
376 "archive_private": False,
377@@ -243,6 +261,7 @@
378 "name": u"test-snap",
379 "proxy_url": self.proxy_url,
380 "revocation_endpoint": self.revocation_endpoint,
381+ "trusted_keys": expected_trusted_keys,
382 }, args)
383
384 @defer.inlineCallbacks
385@@ -252,8 +271,9 @@
386 [ref] = self.factory.makeGitRefs()
387 removeSecurityProxy(ref.repository)._default_branch = ref.path
388 job = self.makeJob(git_ref=ref.repository.getRefByPath(u"HEAD"))
389- expected_archives = get_sources_list_for_building(
390- job.build, job.build.distro_arch_series, None)
391+ expected_archives, expected_trusted_keys = (
392+ yield get_sources_list_for_building(
393+ job.build, job.build.distro_arch_series, None))
394 args = yield job._extraBuildArgs()
395 self.assertEqual({
396 "archive_private": False,
397@@ -263,6 +283,7 @@
398 "name": u"test-snap",
399 "proxy_url": self.proxy_url,
400 "revocation_endpoint": self.revocation_endpoint,
401+ "trusted_keys": expected_trusted_keys,
402 }, args)
403
404 @defer.inlineCallbacks
405@@ -273,8 +294,9 @@
406 ref = self.factory.makeGitRefRemote(
407 repository_url=url, path=u"refs/heads/master")
408 job = self.makeJob(git_ref=ref)
409- expected_archives = yield get_sources_list_for_building(
410- job.build, job.build.distro_arch_series, None)
411+ expected_archives, expected_trusted_keys = (
412+ yield get_sources_list_for_building(
413+ job.build, job.build.distro_arch_series, None))
414 args = yield job._extraBuildArgs()
415 self.assertEqual({
416 "archive_private": False,
417@@ -285,6 +307,7 @@
418 "name": u"test-snap",
419 "proxy_url": self.proxy_url,
420 "revocation_endpoint": self.revocation_endpoint,
421+ "trusted_keys": expected_trusted_keys,
422 }, args)
423
424 @defer.inlineCallbacks
425@@ -294,8 +317,9 @@
426 url = u"https://git.example.org/foo"
427 ref = self.factory.makeGitRefRemote(repository_url=url, path=u"HEAD")
428 job = self.makeJob(git_ref=ref)
429- expected_archives = get_sources_list_for_building(
430- job.build, job.build.distro_arch_series, None)
431+ expected_archives, expected_trusted_keys = (
432+ yield get_sources_list_for_building(
433+ job.build, job.build.distro_arch_series, None))
434 args = yield job._extraBuildArgs()
435 self.assertEqual({
436 "archive_private": False,
437@@ -305,10 +329,29 @@
438 "name": u"test-snap",
439 "proxy_url": self.proxy_url,
440 "revocation_endpoint": self.revocation_endpoint,
441+ "trusted_keys": expected_trusted_keys,
442 }, args)
443
444 @defer.inlineCallbacks
445- def test_extraBuildArgs_proxy_url_set(self):
446+ def test_extraBuildArgs_archive_trusted_keys(self):
447+ # If the archive has a signing key, _extraBuildArgs sends it.
448+ yield self.useFixture(InProcessKeyServerFixture()).start()
449+ archive = self.factory.makeArchive()
450+ key_path = os.path.join(gpgkeysdir, "ppa-sample@canonical.com.sec")
451+ yield IArchiveSigningKey(archive).setSigningKey(
452+ key_path, async_keyserver=True)
453+ job = self.makeJob(archive=archive)
454+ self.factory.makeBinaryPackagePublishingHistory(
455+ distroarchseries=job.build.distro_arch_series,
456+ pocket=job.build.pocket, archive=archive,
457+ status=PackagePublishingStatus.PUBLISHED)
458+ args = yield job._extraBuildArgs()
459+ self.assertThat(args["trusted_keys"], MatchesListwise([
460+ Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"),
461+ ]))
462+
463+ @defer.inlineCallbacks
464+ def test_composeBuildRequest_proxy_url_set(self):
465 job = self.makeJob()
466 build_request = yield job.composeBuildRequest(None)
467 proxy_url = ("http://{username}:{password}"
468
469=== modified file 'lib/lp/soyuz/adapters/archivedependencies.py'
470--- lib/lp/soyuz/adapters/archivedependencies.py 2016-04-07 00:04:42 +0000
471+++ lib/lp/soyuz/adapters/archivedependencies.py 2017-06-13 17:24:41 +0000
472@@ -1,4 +1,4 @@
473-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
474+# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
475 # GNU Affero General Public License version 3 (see the file LICENSE).
476
477 """Archive dependencies helper function.
478@@ -36,11 +36,15 @@
479 'pocket_dependencies',
480 ]
481
482+import base64
483 import logging
484 import traceback
485
486 from lazr.uri import URI
487+from twisted.internet import defer
488+from twisted.internet.threads import deferToThread
489 from zope.component import getUtility
490+from zope.security.proxy import removeSecurityProxy
491
492 from lp.app.errors import NotFoundError
493 from lp.registry.interfaces.distroseriesparent import IDistroSeriesParentSet
494@@ -48,6 +52,11 @@
495 PackagePublishingPocket,
496 pocketsuffix,
497 )
498+from lp.services.gpg.interfaces import (
499+ GPGKeyNotFoundError,
500+ IGPGHandler,
501+ )
502+from lp.services.timeout import default_timeout
503 from lp.soyuz.enums import (
504 ArchivePurpose,
505 PackagePublishingStatus,
506@@ -229,16 +238,19 @@
507 return deps
508
509
510+@defer.inlineCallbacks
511 def get_sources_list_for_building(build, distroarchseries, sourcepackagename,
512 tools_source=None, logger=None):
513- """Return the sources_list entries required to build the given item.
514+ """Return sources.list entries and keys required to build the given item.
515
516- The entries are returned in the order that is most useful;
517+ The sources.list entries are returned in the order that is most useful:
518 1. the context archive itself
519 2. external dependencies
520 3. user-selected archive dependencies
521 4. the default primary archive
522
523+ The keys are in an arbitrary order.
524+
525 :param build: a context `IBuild`.
526 :param distroarchseries: A `IDistroArchSeries`
527 :param sourcepackagename: A source package name (as text)
528@@ -246,14 +258,16 @@
529 additional dependency for build tools, just before the default
530 primary archive.
531 :param logger: an optional logger.
532- :return: a deb sources_list entries (lines).
533+ :return: a Deferred resolving to a tuple containing a list of deb
534+ sources.list entries (lines) and a list of base64-encoded public
535+ keys.
536 """
537 deps = expand_dependencies(
538 build.archive, distroarchseries, build.pocket,
539 build.current_component, sourcepackagename,
540 tools_source=tools_source, logger=logger)
541- sources_list_lines = \
542- _get_sources_list_for_dependencies(deps)
543+ sources_list_lines, trusted_keys = (
544+ yield _get_sources_list_for_dependencies(deps, logger=logger))
545
546 external_dep_lines = []
547 # Append external sources.list lines for this build if specified. No
548@@ -289,8 +303,9 @@
549 # binaries that need to override primary binaries of the same
550 # version), we want the external dependency lines to show up second:
551 # after the archive itself, but before any other dependencies.
552- return [sources_list_lines[0]] + external_dep_lines + \
553- sources_list_lines[1:]
554+ defer.returnValue(
555+ ([sources_list_lines[0]] + external_dep_lines + sources_list_lines[1:],
556+ trusted_keys))
557
558
559 def _has_published_binaries(archive, distroarchseries, pocket):
560@@ -323,8 +338,9 @@
561 return 'deb %s %s %s' % (url, suite, ' '.join(components))
562
563
564-def _get_sources_list_for_dependencies(dependencies):
565- """Return a list of sources_list lines.
566+@defer.inlineCallbacks
567+def _get_sources_list_for_dependencies(dependencies, logger=None):
568+ """Return sources.list entries and keys.
569
570 Process the given list of dependency tuples for the given
571 `DistroArchseries`.
572@@ -334,9 +350,15 @@
573 list of `IComponent` names)
574 :param distroarchseries: target `IDistroArchSeries`;
575
576- :return: a list of sources_list formatted lines.
577+ :return: a tuple containing a list of sources.list formatted lines and a
578+ list of base64-encoded public keys.
579 """
580 sources_list_lines = []
581+ trusted_keys = {}
582+ # The handler's security proxying doesn't protect anything useful here,
583+ # and the thread that we defer key retrieval to doesn't have an
584+ # interaction.
585+ gpghandler = removeSecurityProxy(getUtility(IGPGHandler))
586 for dep in dependencies:
587 if isinstance(dep, basestring):
588 sources_list_lines.append(dep)
589@@ -356,8 +378,27 @@
590 sources_list_line = _get_binary_sources_list_line(
591 archive, distro_arch_series, pocket, components)
592 sources_list_lines.append(sources_list_line)
593-
594- return sources_list_lines
595+ fingerprint = archive.signing_key_fingerprint
596+ if fingerprint is not None and fingerprint not in trusted_keys:
597+ def get_key():
598+ with default_timeout(15.0):
599+ try:
600+ return gpghandler.retrieveKey(fingerprint)
601+ except GPGKeyNotFoundError as e:
602+ # For now, just log this and proceed without the
603+ # key. We'll have to fix any outstanding cases
604+ # of this before we can switch to requiring
605+ # authentication across the board.
606+ if logger is not None:
607+ logger.warning(str(e))
608+ return None
609+
610+ key = yield deferToThread(get_key)
611+ if key is not None:
612+ trusted_keys[fingerprint] = base64.b64encode(key.export())
613+
614+ defer.returnValue(
615+ (sources_list_lines, [v for k, v in sorted(trusted_keys.items())]))
616
617
618 def _get_default_primary_dependencies(archive, distro_arch_series, component,
619
620=== modified file 'lib/lp/soyuz/adapters/tests/test_archivedependencies.py'
621--- lib/lp/soyuz/adapters/tests/test_archivedependencies.py 2017-04-25 11:36:10 +0000
622+++ lib/lp/soyuz/adapters/tests/test_archivedependencies.py 2017-06-13 17:24:41 +0000
623@@ -7,12 +7,23 @@
624
625 __metaclass__ = type
626
627-from testtools.matchers import StartsWith
628+import os.path
629+
630+from testtools.deferredruntest import AsynchronousDeferredRunTest
631+from testtools.matchers import (
632+ MatchesSetwise,
633+ StartsWith,
634+ )
635 import transaction
636+from twisted.internet import defer
637 from zope.component import getUtility
638
639+from lp.archivepublisher.interfaces.archivesigningkey import (
640+ IArchiveSigningKey,
641+ )
642 from lp.registry.interfaces.distribution import IDistributionSet
643 from lp.registry.interfaces.pocket import PackagePublishingPocket
644+from lp.services.config import config
645 from lp.services.log.logger import BufferLogger
646 from lp.soyuz.adapters.archivedependencies import (
647 default_component_dependency_name,
648@@ -25,8 +36,11 @@
649 from lp.soyuz.enums import PackagePublishingStatus
650 from lp.soyuz.interfaces.archive import IArchive
651 from lp.soyuz.interfaces.component import IComponentSet
652+from lp.soyuz.tests.soyuz import Base64KeyMatches
653 from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
654 from lp.testing import TestCaseWithFactory
655+from lp.testing.gpgkeys import gpgkeysdir
656+from lp.testing.keyserver import InProcessKeyServerFixture
657 from lp.testing.layers import (
658 LaunchpadZopelessLayer,
659 ZopelessDatabaseLayer,
660@@ -111,10 +125,17 @@
661 """Test sources.list contents for building, and related mechanisms."""
662
663 layer = LaunchpadZopelessLayer
664+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=10)
665
666 ubuntu_components = [
667 "main", "restricted", "universe", "multiverse", "partner"]
668
669+ fingerprints = {
670+ "ppa-sample@canonical.com": "0D57E99656BEFB0897606EE9A022DD1F5001B46D",
671+ "ppa-sample-4096@canonical.com": (
672+ "B7B1966662BA8D3F5A6ED89BD640F4A593B2CF67"),
673+ }
674+
675 def setUp(self):
676 super(TestSourcesList, self).setUp()
677 self.publisher = SoyuzTestPublisher()
678@@ -140,12 +161,22 @@
679 PackagePublishingPocket.UPDATES),
680 pocket_dependencies[default_pocket_dependency])
681
682- def makeArchive(self, publish_binary=False, **kwargs):
683+ @defer.inlineCallbacks
684+ def makeArchive(self, signing_key_name="ppa-sample@canonical.com",
685+ publish_binary=False, **kwargs):
686+ try:
687+ getattr(config, "in-process-key-server-fixture")
688+ except AttributeError:
689+ yield self.useFixture(InProcessKeyServerFixture()).start()
690 archive = self.factory.makeArchive(distribution=self.ubuntu, **kwargs)
691+ if signing_key_name is not None:
692+ key_path = os.path.join(gpgkeysdir, "%s.sec" % signing_key_name)
693+ yield IArchiveSigningKey(archive).setSigningKey(
694+ key_path, async_keyserver=True)
695 if publish_binary:
696 self.publisher.getPubBinaries(
697 archive=archive, status=PackagePublishingStatus.PUBLISHED)
698- return archive
699+ defer.returnValue(archive)
700
701 def makeBuild(self, **kwargs):
702 pub_source = self.publisher.getPubSource(**kwargs)
703@@ -159,77 +190,90 @@
704 build.archive, build.distro_series,
705 build.source_package_release.name).name)
706
707- def assertSourcesList(self, expected, build, **kwargs):
708+ @defer.inlineCallbacks
709+ def assertSourcesListAndKeys(self, expected_sources_list,
710+ expected_key_names, build, **kwargs):
711 expected_lines = []
712- for archive_or_prefix, suffixes in expected:
713+ for archive_or_prefix, suffixes in expected_sources_list:
714 if IArchive.providedBy(archive_or_prefix):
715 prefix = "deb %s " % archive_or_prefix.archive_url
716 else:
717 prefix = archive_or_prefix + " "
718 expected_lines.extend([prefix + suffix for suffix in suffixes])
719- sources_list = get_sources_list_for_building(
720+ sources_list, trusted_keys = yield get_sources_list_for_building(
721 build, build.distro_arch_series, build.source_package_release.name,
722 **kwargs)
723 self.assertEqual(expected_lines, sources_list)
724+ key_matchers = [
725+ Base64KeyMatches(self.fingerprints[key_name])
726+ for key_name in expected_key_names]
727+ self.assertThat(trusted_keys, MatchesSetwise(*key_matchers))
728
729+ @defer.inlineCallbacks
730 def test_ppa_with_no_binaries(self):
731 # If there are no published binaries in a PPA, only its primary
732 # archive dependencies need to be considered.
733- ppa = self.makeArchive()
734+ ppa = yield self.makeArchive()
735 build = self.makeBuild(archive=ppa)
736 self.assertEqual(
737 0, ppa.getAllPublishedBinaries(
738 distroarchseries=build.distro_arch_series,
739 status=PackagePublishingStatus.PUBLISHED).count())
740- self.assertSourcesList(
741+ yield self.assertSourcesListAndKeys(
742 [(self.ubuntu.main_archive, [
743 "hoary main restricted universe multiverse",
744 "hoary-security main restricted universe multiverse",
745 "hoary-updates main restricted universe multiverse",
746 ]),
747- ], build)
748+ ], [], build)
749
750+ @defer.inlineCallbacks
751 def test_ppa_with_binaries(self):
752 # If there are binaries published in a PPA, then the PPA is
753 # considered as well as its primary dependencies.
754- ppa = self.makeArchive(publish_binary=True)
755+ ppa = yield self.makeArchive(publish_binary=True)
756 build = self.makeBuild(archive=ppa)
757- self.assertSourcesList(
758+ yield self.assertSourcesListAndKeys(
759 [(ppa, ["hoary main"]),
760 (self.ubuntu.main_archive, [
761 "hoary main restricted universe multiverse",
762 "hoary-security main restricted universe multiverse",
763 "hoary-updates main restricted universe multiverse",
764 ]),
765- ], build)
766+ ], ["ppa-sample@canonical.com"], build)
767
768+ @defer.inlineCallbacks
769 def test_dependent_ppa_with_no_binaries(self):
770 # A depended-upon PPA is not considered if it has no published
771 # binaries.
772- lower_ppa = self.makeArchive()
773- upper_ppa = self.makeArchive(publish_binary=True)
774+ lower_ppa = yield self.makeArchive(
775+ signing_key_name="ppa-sample-4096@canonical.com")
776+ upper_ppa = yield self.makeArchive(publish_binary=True)
777 upper_ppa.addArchiveDependency(
778 lower_ppa, PackagePublishingPocket.RELEASE,
779 getUtility(IComponentSet)["main"])
780 build = self.makeBuild(archive=upper_ppa)
781- self.assertSourcesList(
782+ yield self.assertSourcesListAndKeys(
783 [(upper_ppa, ["hoary main"]),
784 (self.ubuntu.main_archive, [
785 "hoary main restricted universe multiverse",
786 "hoary-security main restricted universe multiverse",
787 "hoary-updates main restricted universe multiverse",
788 ]),
789- ], build)
790+ ], ["ppa-sample@canonical.com"], build)
791
792+ @defer.inlineCallbacks
793 def test_dependent_ppa_with_binaries(self):
794 # A depended-upon PPA is considered if it has published binaries.
795- lower_ppa = self.makeArchive(publish_binary=True)
796- upper_ppa = self.makeArchive(publish_binary=True)
797+ lower_ppa = yield self.makeArchive(
798+ signing_key_name="ppa-sample-4096@canonical.com",
799+ publish_binary=True)
800+ upper_ppa = yield self.makeArchive(publish_binary=True)
801 upper_ppa.addArchiveDependency(
802 lower_ppa, PackagePublishingPocket.RELEASE,
803 getUtility(IComponentSet)["main"])
804 build = self.makeBuild(archive=upper_ppa)
805- self.assertSourcesList(
806+ yield self.assertSourcesListAndKeys(
807 [(upper_ppa, ["hoary main"]),
808 (lower_ppa, ["hoary main"]),
809 (self.ubuntu.main_archive, [
810@@ -237,14 +281,19 @@
811 "hoary-security main restricted universe multiverse",
812 "hoary-updates main restricted universe multiverse",
813 ]),
814- ], build)
815+ ],
816+ ["ppa-sample@canonical.com", "ppa-sample-4096@canonical.com"],
817+ build)
818
819+ @defer.inlineCallbacks
820 def test_lax_supported_component_dependencies(self):
821 # Dependencies for series with
822 # strict_supported_component_dependencies=False are reasonable.
823 # PPAs only have the "main" component.
824- lower_ppa = self.makeArchive(publish_binary=True)
825- upper_ppa = self.makeArchive(publish_binary=True)
826+ lower_ppa = yield self.makeArchive(
827+ signing_key_name="ppa-sample-4096@canonical.com",
828+ publish_binary=True)
829+ upper_ppa = yield self.makeArchive(publish_binary=True)
830 upper_ppa.addArchiveDependency(
831 lower_ppa, PackagePublishingPocket.RELEASE,
832 getUtility(IComponentSet)["main"])
833@@ -252,7 +301,7 @@
834 self.ubuntu.main_archive, PackagePublishingPocket.UPDATES,
835 getUtility(IComponentSet)["restricted"])
836 build = self.makeBuild(archive=upper_ppa)
837- self.assertSourcesList(
838+ yield self.assertSourcesListAndKeys(
839 [(upper_ppa, ["hoary main"]),
840 (lower_ppa, ["hoary main"]),
841 (self.ubuntu.main_archive, [
842@@ -260,10 +309,12 @@
843 "hoary-security main restricted",
844 "hoary-updates main restricted",
845 ]),
846- ], build)
847+ ],
848+ ["ppa-sample@canonical.com", "ppa-sample-4096@canonical.com"],
849+ build)
850 self.hoary.strict_supported_component_dependencies = False
851 transaction.commit()
852- self.assertSourcesList(
853+ yield self.assertSourcesListAndKeys(
854 [(upper_ppa, ["hoary main"]),
855 (lower_ppa, ["hoary main"]),
856 (self.ubuntu.main_archive, [
857@@ -271,41 +322,45 @@
858 "hoary-security main restricted universe multiverse",
859 "hoary-updates main restricted universe multiverse",
860 ]),
861- ], build)
862+ ],
863+ ["ppa-sample@canonical.com", "ppa-sample-4096@canonical.com"],
864+ build)
865
866+ @defer.inlineCallbacks
867 def test_no_op_primary_archive_dependency(self):
868 # Overriding the default primary archive dependencies with exactly
869 # the same values has no effect.
870- ppa = self.makeArchive()
871+ ppa = yield self.makeArchive()
872 ppa.addArchiveDependency(
873 self.ubuntu.main_archive, PackagePublishingPocket.UPDATES,
874 getUtility(IComponentSet)["multiverse"])
875 build = self.makeBuild(archive=ppa)
876- self.assertSourcesList(
877+ yield self.assertSourcesListAndKeys(
878 [(self.ubuntu.main_archive, [
879 "hoary main restricted universe multiverse",
880 "hoary-security main restricted universe multiverse",
881 "hoary-updates main restricted universe multiverse",
882 ]),
883- ], build)
884+ ], [], build)
885
886+ @defer.inlineCallbacks
887 def test_primary_archive_dependency_security(self):
888 # The primary archive dependency can be modified to behave as an
889 # embargoed archive that builds security updates. This is done by
890 # setting the SECURITY pocket dependencies (RELEASE and SECURITY)
891 # and following the component dependencies of the component where
892 # the source was last published in the primary archive.
893- ppa = self.makeArchive()
894+ ppa = yield self.makeArchive()
895 ppa.addArchiveDependency(
896 self.ubuntu.main_archive, PackagePublishingPocket.SECURITY)
897 build = self.makeBuild(archive=ppa)
898 self.assertPrimaryCurrentComponent("universe", build)
899- self.assertSourcesList(
900+ yield self.assertSourcesListAndKeys(
901 [(self.ubuntu.main_archive, [
902 "hoary main universe",
903 "hoary-security main universe",
904 ]),
905- ], build)
906+ ], [], build)
907 self.publisher.getPubSource(
908 sourcename="with-ancestry", version="1.0",
909 archive=self.ubuntu.main_archive)
910@@ -313,59 +368,63 @@
911 sourcename="with-ancestry", version="1.1",
912 archive=ppa).createMissingBuilds()
913 self.assertPrimaryCurrentComponent("main", build_with_ancestry)
914- self.assertSourcesList(
915+ yield self.assertSourcesListAndKeys(
916 [(self.ubuntu.main_archive, [
917 "hoary main",
918 "hoary-security main",
919 ]),
920- ], build_with_ancestry)
921+ ], [], build_with_ancestry)
922
923+ @defer.inlineCallbacks
924 def test_primary_archive_dependency_release(self):
925 # The primary archive dependency can be modified to behave as a
926 # pristine build environment based only on what was included in the
927 # original release of the corresponding series.
928- ppa = self.makeArchive()
929+ ppa = yield self.makeArchive()
930 ppa.addArchiveDependency(
931 self.ubuntu.main_archive, PackagePublishingPocket.RELEASE,
932 getUtility(IComponentSet)["restricted"])
933 build = self.makeBuild(archive=ppa)
934- self.assertSourcesList(
935- [(self.ubuntu.main_archive, ["hoary main restricted"])], build)
936+ yield self.assertSourcesListAndKeys(
937+ [(self.ubuntu.main_archive, ["hoary main restricted"])], [], build)
938
939+ @defer.inlineCallbacks
940 def test_primary_archive_dependency_proposed(self):
941 # The primary archive dependency can be modified to extend the build
942 # environment for PROPOSED.
943- ppa = self.makeArchive()
944+ ppa = yield self.makeArchive()
945 ppa.addArchiveDependency(
946 self.ubuntu.main_archive, PackagePublishingPocket.PROPOSED,
947 getUtility(IComponentSet)["multiverse"])
948 build = self.makeBuild(archive=ppa)
949- self.assertSourcesList(
950+ yield self.assertSourcesListAndKeys(
951 [(self.ubuntu.main_archive, [
952 "hoary main restricted universe multiverse",
953 "hoary-security main restricted universe multiverse",
954 "hoary-updates main restricted universe multiverse",
955 "hoary-proposed main restricted universe multiverse",
956 ]),
957- ], build)
958+ ], [], build)
959
960+ @defer.inlineCallbacks
961 def test_primary_archive_dependency_backports(self):
962 # The primary archive dependency can be modified to extend the build
963 # environment for PROPOSED.
964- ppa = self.makeArchive()
965+ ppa = yield self.makeArchive()
966 ppa.addArchiveDependency(
967 self.ubuntu.main_archive, PackagePublishingPocket.BACKPORTS,
968 getUtility(IComponentSet)["multiverse"])
969 build = self.makeBuild(archive=ppa)
970- self.assertSourcesList(
971+ yield self.assertSourcesListAndKeys(
972 [(self.ubuntu.main_archive, [
973 "hoary main restricted universe multiverse",
974 "hoary-security main restricted universe multiverse",
975 "hoary-updates main restricted universe multiverse",
976 "hoary-backports main restricted universe multiverse",
977 ]),
978- ], build)
979+ ], [], build)
980
981+ @defer.inlineCallbacks
982 def test_partner(self):
983 # Similarly to what happens with PPA builds, partner builds may
984 # depend on any component in the primary archive. This behaviour
985@@ -377,15 +436,16 @@
986 archive=partner, component="partner",
987 status=PackagePublishingStatus.PUBLISHED)
988 build = self.makeBuild(archive=partner, component="partner")
989- self.assertSourcesList(
990+ yield self.assertSourcesListAndKeys(
991 [(partner, ["hoary partner"]),
992 (primary, [
993 "hoary main restricted universe multiverse",
994 "hoary-security main restricted universe multiverse",
995 "hoary-updates main restricted universe multiverse",
996 ]),
997- ], build)
998+ ], [], build)
999
1000+ @defer.inlineCallbacks
1001 def test_partner_proposed(self):
1002 # The partner archive's PROPOSED pocket builds against itself, but
1003 # still uses the default UPDATES dependency for the primary archive
1004@@ -401,7 +461,7 @@
1005 build = self.makeBuild(
1006 archive=partner, component="partner",
1007 pocket=PackagePublishingPocket.PROPOSED)
1008- self.assertSourcesList(
1009+ yield self.assertSourcesListAndKeys(
1010 [(partner, [
1011 "hoary partner",
1012 "hoary-proposed partner",
1013@@ -411,19 +471,20 @@
1014 "hoary-security main restricted universe multiverse",
1015 "hoary-updates main restricted universe multiverse",
1016 ]),
1017- ], build)
1018+ ], [], build)
1019
1020+ @defer.inlineCallbacks
1021 def test_archive_external_dependencies(self):
1022 # An archive can be manually given additional external dependencies.
1023 # If present, "%(series)s" is replaced with the series name for the
1024 # build being dispatched.
1025- ppa = self.makeArchive(publish_binary=True)
1026+ ppa = yield self.makeArchive(publish_binary=True)
1027 ppa.external_dependencies = (
1028 "deb http://user:pass@repository zoing everything\n"
1029 "deb http://user:pass@repository %(series)s public private\n"
1030 "deb http://user:pass@repository %(series)s-extra public")
1031 build = self.makeBuild(archive=ppa)
1032- self.assertSourcesList(
1033+ yield self.assertSourcesListAndKeys(
1034 [(ppa, ["hoary main"]),
1035 ("deb http://user:pass@repository", [
1036 "zoing everything",
1037@@ -435,16 +496,17 @@
1038 "hoary-security main restricted universe multiverse",
1039 "hoary-updates main restricted universe multiverse",
1040 ]),
1041- ], build)
1042+ ], ["ppa-sample@canonical.com"], build)
1043
1044+ @defer.inlineCallbacks
1045 def test_build_external_dependencies(self):
1046 # A single build can be manually given additional external
1047 # dependencies.
1048- ppa = self.makeArchive(publish_binary=True)
1049+ ppa = yield self.makeArchive(publish_binary=True)
1050 build = self.makeBuild(archive=ppa)
1051 build.api_external_dependencies = (
1052 "deb http://user:pass@repository foo bar")
1053- self.assertSourcesList(
1054+ yield self.assertSourcesListAndKeys(
1055 [(ppa, ["hoary main"]),
1056 ("deb http://user:pass@repository", ["foo bar"]),
1057 (self.ubuntu.main_archive, [
1058@@ -452,14 +514,15 @@
1059 "hoary-security main restricted universe multiverse",
1060 "hoary-updates main restricted universe multiverse",
1061 ]),
1062- ], build)
1063+ ], ["ppa-sample@canonical.com"], build)
1064
1065+ @defer.inlineCallbacks
1066 def test_build_tools(self):
1067 # We can force an extra build tools line to be added to
1068 # sources.list, which is useful for specialised build types.
1069- ppa = self.makeArchive(publish_binary=True)
1070+ ppa = yield self.makeArchive(publish_binary=True)
1071 build = self.makeBuild(archive=ppa)
1072- self.assertSourcesList(
1073+ yield self.assertSourcesListAndKeys(
1074 [(ppa, ["hoary main"]),
1075 ("deb http://example.org", ["hoary main"]),
1076 (self.ubuntu.main_archive, [
1077@@ -467,15 +530,18 @@
1078 "hoary-security main restricted universe multiverse",
1079 "hoary-updates main restricted universe multiverse",
1080 ]),
1081- ], build, tools_source="deb http://example.org %(series)s main")
1082+ ],
1083+ ["ppa-sample@canonical.com"], build,
1084+ tools_source="deb http://example.org %(series)s main")
1085
1086+ @defer.inlineCallbacks
1087 def test_build_tools_bad_formatting(self):
1088 # If tools_source is badly formatted, we log the error but don't
1089 # blow up. (Note the missing "s" at the end of "%(series)".)
1090- ppa = self.makeArchive(publish_binary=True)
1091+ ppa = yield self.makeArchive(publish_binary=True)
1092 build = self.makeBuild(archive=ppa)
1093 logger = BufferLogger()
1094- self.assertSourcesList(
1095+ yield self.assertSourcesListAndKeys(
1096 [(ppa, ["hoary main"]),
1097 (self.ubuntu.main_archive, [
1098 "hoary main restricted universe multiverse",
1099@@ -483,11 +549,13 @@
1100 "hoary-updates main restricted universe multiverse",
1101 ]),
1102 ],
1103- build, tools_source="deb http://example.org %(series) main",
1104+ ["ppa-sample@canonical.com"], build,
1105+ tools_source="deb http://example.org %(series) main",
1106 logger=logger)
1107 self.assertThat(logger.getLogBuffer(), StartsWith(
1108 "ERROR Exception processing build tools sources.list entry:\n"))
1109
1110+ @defer.inlineCallbacks
1111 def test_overlay(self):
1112 # An overlay distroseries is a derived distribution which works like
1113 # a PPA. This means that the parent's details gets added to the
1114@@ -508,10 +576,10 @@
1115 pocket=PackagePublishingPocket.SECURITY,
1116 component=getUtility(IComponentSet)["universe"])
1117 build = self.makeBuild()
1118- self.assertSourcesList(
1119+ yield self.assertSourcesListAndKeys(
1120 [(self.ubuntu.main_archive, ["hoary main"]),
1121 (depdistro.main_archive, [
1122 "depseries main universe",
1123 "depseries-security main universe",
1124 ]),
1125- ], build)
1126+ ], [], build)
1127
1128=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehaviour.py'
1129--- lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2017-04-28 17:30:55 +0000
1130+++ lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2017-06-13 17:24:41 +0000
1131@@ -164,8 +164,9 @@
1132 args["ogrecomponent"] = (
1133 build.current_component.name)
1134
1135- args['archives'] = yield get_sources_list_for_building(
1136- build, das, build.source_package_release.name, logger=logger)
1137+ args['archives'], args['trusted_keys'] = (
1138+ yield get_sources_list_for_building(
1139+ build, das, build.source_package_release.name, logger=logger))
1140 args['archive_private'] = build.archive.private
1141 args['build_debug_symbols'] = build.archive.build_debug_symbols
1142
1143
1144=== modified file 'lib/lp/soyuz/model/livefsbuildbehaviour.py'
1145--- lib/lp/soyuz/model/livefsbuildbehaviour.py 2017-04-28 17:30:55 +0000
1146+++ lib/lp/soyuz/model/livefsbuildbehaviour.py 2017-06-13 17:24:41 +0000
1147@@ -90,8 +90,9 @@
1148 args["pocket"] = build.pocket.name.lower()
1149 args["arch_tag"] = build.distro_arch_series.architecturetag
1150 args["datestamp"] = build.version
1151- args["archives"] = yield get_sources_list_for_building(
1152- build, build.distro_arch_series, None, logger=logger)
1153+ args["archives"], args["trusted_keys"] = (
1154+ yield get_sources_list_for_building(
1155+ build, build.distro_arch_series, None, logger=logger))
1156 args["archive_private"] = build.archive.private
1157 defer.returnValue(args)
1158
1159
1160=== modified file 'lib/lp/soyuz/tests/soyuz.py'
1161--- lib/lp/soyuz/tests/soyuz.py 2015-09-28 17:38:45 +0000
1162+++ lib/lp/soyuz/tests/soyuz.py 2017-06-13 17:24:41 +0000
1163@@ -1,4 +1,4 @@
1164-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
1165+# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
1166 # GNU Affero General Public License version 3 (see the file LICENSE).
1167
1168 """Helper functions/classes for Soyuz tests."""
1169@@ -6,14 +6,22 @@
1170 __metaclass__ = type
1171
1172 __all__ = [
1173+ 'Base64KeyMatches',
1174 'SoyuzTestHelper',
1175 ]
1176
1177+import base64
1178+
1179+from testtools.matchers import (
1180+ Equals,
1181+ Matcher,
1182+ )
1183 from zope.component import getUtility
1184
1185 from lp.registry.interfaces.distribution import IDistributionSet
1186 from lp.registry.interfaces.person import IPersonSet
1187 from lp.registry.interfaces.pocket import PackagePublishingPocket
1188+from lp.services.gpg.interfaces import IGPGHandler
1189 from lp.soyuz.enums import PackagePublishingStatus
1190 from lp.soyuz.model.publishing import SourcePackagePublishingHistory
1191 from lp.testing.sampledata import (
1192@@ -94,3 +102,15 @@
1193 Return True if the lists matches, otherwise False.
1194 """
1195 return [p.id for p in expected] == [r.id for r in given]
1196+
1197+
1198+class Base64KeyMatches(Matcher):
1199+ """Matches if base64-encoded key material has a given fingerprint."""
1200+
1201+ def __init__(self, fingerprint):
1202+ self.fingerprint = fingerprint
1203+
1204+ def match(self, encoded_key):
1205+ key = base64.b64decode(encoded_key)
1206+ return Equals(self.fingerprint).match(
1207+ getUtility(IGPGHandler).importPublicKey(key).fingerprint)
1208
1209=== modified file 'lib/lp/soyuz/tests/test_archive.py'
1210--- lib/lp/soyuz/tests/test_archive.py 2017-04-28 17:30:55 +0000
1211+++ lib/lp/soyuz/tests/test_archive.py 2017-06-13 17:24:41 +0000
1212@@ -9,6 +9,7 @@
1213 timedelta,
1214 )
1215 import doctest
1216+import os.path
1217
1218 from pytz import UTC
1219 from testtools.deferredruntest import AsynchronousDeferredRunTest
1220@@ -16,6 +17,7 @@
1221 AllMatch,
1222 DocTestMatches,
1223 LessThan,
1224+ MatchesListwise,
1225 MatchesPredicate,
1226 MatchesRegex,
1227 MatchesStructure,
1228@@ -29,6 +31,9 @@
1229
1230 from lp.app.errors import NotFoundError
1231 from lp.app.interfaces.launchpad import ILaunchpadCelebrities
1232+from lp.archivepublisher.interfaces.archivesigningkey import (
1233+ IArchiveSigningKey,
1234+ )
1235 from lp.buildmaster.enums import (
1236 BuildQueueStatus,
1237 BuildStatus,
1238@@ -108,6 +113,7 @@
1239 BinaryPackageReleaseDownloadCount,
1240 )
1241 from lp.soyuz.model.component import ComponentSelection
1242+from lp.soyuz.tests.soyuz import Base64KeyMatches
1243 from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
1244 from lp.testing import (
1245 admin_logged_in,
1246@@ -120,6 +126,8 @@
1247 StormStatementRecorder,
1248 TestCaseWithFactory,
1249 )
1250+from lp.testing.gpgkeys import gpgkeysdir
1251+from lp.testing.keyserver import InProcessKeyServerFixture
1252 from lp.testing.layers import (
1253 DatabaseFunctionalLayer,
1254 LaunchpadFunctionalLayer,
1255@@ -1748,12 +1756,17 @@
1256 class TestArchiveDependencies(TestCaseWithFactory):
1257
1258 layer = LaunchpadZopelessLayer
1259- run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=30)
1260+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=10)
1261
1262 @defer.inlineCallbacks
1263 def test_private_sources_list(self):
1264 """Entries for private dependencies include credentials."""
1265 p3a = self.factory.makeArchive(name='p3a', private=True)
1266+ with InProcessKeyServerFixture() as keyserver:
1267+ yield keyserver.start()
1268+ key_path = os.path.join(gpgkeysdir, 'ppa-sample@canonical.com.sec')
1269+ yield IArchiveSigningKey(p3a).setSigningKey(
1270+ key_path, async_keyserver=True)
1271 dependency = self.factory.makeArchive(
1272 name='dependency', private=True, owner=p3a.owner)
1273 with person_logged_in(p3a.owner):
1274@@ -1764,13 +1777,16 @@
1275 PackagePublishingPocket.RELEASE)
1276 build = self.factory.makeBinaryPackageBuild(archive=p3a,
1277 distroarchseries=bpph.distroarchseries)
1278- sources_list = yield get_sources_list_for_building(
1279+ sources_list, trusted_keys = yield get_sources_list_for_building(
1280 build, build.distro_arch_series,
1281 build.source_package_release.name)
1282 matches = MatchesRegex(
1283 "deb http://buildd:sekrit@private-ppa.launchpad.dev/"
1284 "person-name-.*/dependency/ubuntu distroseries-.* main")
1285 self.assertThat(sources_list[0], matches)
1286+ self.assertThat(trusted_keys, MatchesListwise([
1287+ Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"),
1288+ ]))
1289
1290 def test_invalid_external_dependencies(self):
1291 """Trying to set invalid external dependencies raises an exception."""
1292@@ -2018,7 +2034,7 @@
1293 self._createDep(
1294 test_publisher, series11, 'series12', 'depdistro4', 'multiverse',
1295 PackagePublishingPocket.UPDATES)
1296- sources_list = yield get_sources_list_for_building(
1297+ sources_list, trusted_keys = yield get_sources_list_for_building(
1298 build, build.distro_arch_series, build.source_package_release.name)
1299
1300 self.assertThat(
1301@@ -2040,6 +2056,7 @@
1302 ".../depdistro4 series12-updates "
1303 "main restricted universe multiverse\n",
1304 doctest.ELLIPSIS))
1305+ self.assertEqual([], trusted_keys)
1306
1307
1308 class TestComponents(TestCaseWithFactory):
1309
1310=== modified file 'lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py'
1311--- lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2017-04-28 17:30:55 +0000
1312+++ lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2017-06-13 17:24:41 +0000
1313@@ -12,6 +12,7 @@
1314
1315 from storm.store import Store
1316 from testtools.deferredruntest import AsynchronousDeferredRunTest
1317+from testtools.matchers import MatchesListwise
1318 import transaction
1319 from twisted.internet import defer
1320 from twisted.trial.unittest import TestCase as TrialTestCase
1321@@ -19,6 +20,9 @@
1322 from zope.security.proxy import removeSecurityProxy
1323
1324 from lp.archivepublisher.diskpool import poolify
1325+from lp.archivepublisher.interfaces.archivesigningkey import (
1326+ IArchiveSigningKey,
1327+ )
1328 from lp.buildmaster.enums import (
1329 BuilderCleanStatus,
1330 BuildStatus,
1331@@ -55,9 +59,15 @@
1332 from lp.soyuz.adapters.archivedependencies import (
1333 get_sources_list_for_building,
1334 )
1335-from lp.soyuz.enums import ArchivePurpose
1336+from lp.soyuz.enums import (
1337+ ArchivePurpose,
1338+ PackagePublishingStatus,
1339+ )
1340+from lp.soyuz.tests.soyuz import Base64KeyMatches
1341 from lp.testing import TestCaseWithFactory
1342 from lp.testing.dbuser import switch_dbuser
1343+from lp.testing.gpgkeys import gpgkeysdir
1344+from lp.testing.keyserver import InProcessKeyServerFixture
1345 from lp.testing.layers import LaunchpadZopelessLayer
1346
1347
1348@@ -107,7 +117,7 @@
1349 das = build.distro_arch_series
1350 ds_name = das.distroseries.name
1351 suite = ds_name + pocketsuffix[build.pocket]
1352- archives = yield get_sources_list_for_building(
1353+ archives, trusted_keys = yield get_sources_list_for_building(
1354 build, das, build.source_package_release.name)
1355 arch_indep = das.isNominatedArchIndep
1356 if component is None:
1357@@ -131,6 +141,7 @@
1358 'ogrecomponent': component,
1359 'distribution': das.distroseries.distribution.name,
1360 'suite': suite,
1361+ 'trusted_keys': trusted_keys,
1362 }
1363 build_log = [
1364 ('build', build.build_cookie, 'binarypackage',
1365@@ -324,6 +335,23 @@
1366 extra_args = yield IBuildFarmJobBehaviour(build)._extraBuildArgs(build)
1367 self.assertTrue(extra_args['arch_indep'])
1368
1369+ @defer.inlineCallbacks
1370+ def test_extraBuildArgs_archive_trusted_keys(self):
1371+ # If the archive has a signing key, _extraBuildArgs sends it.
1372+ yield self.useFixture(InProcessKeyServerFixture()).start()
1373+ archive = self.factory.makeArchive()
1374+ key_path = os.path.join(gpgkeysdir, "ppa-sample@canonical.com.sec")
1375+ yield IArchiveSigningKey(archive).setSigningKey(
1376+ key_path, async_keyserver=True)
1377+ build = self.factory.makeBinaryPackageBuild(archive=archive)
1378+ self.factory.makeBinaryPackagePublishingHistory(
1379+ distroarchseries=build.distro_arch_series, pocket=build.pocket,
1380+ archive=archive, status=PackagePublishingStatus.PUBLISHED)
1381+ args = yield IBuildFarmJobBehaviour(build)._extraBuildArgs(build)
1382+ self.assertThat(args["trusted_keys"], MatchesListwise([
1383+ Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"),
1384+ ]))
1385+
1386 def test_verifyBuildRequest(self):
1387 # Don't allow a virtual build on a non-virtual builder.
1388 archive = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
1389
1390=== modified file 'lib/lp/soyuz/tests/test_livefsbuildbehaviour.py'
1391--- lib/lp/soyuz/tests/test_livefsbuildbehaviour.py 2017-04-28 17:30:55 +0000
1392+++ lib/lp/soyuz/tests/test_livefsbuildbehaviour.py 2017-06-13 17:24:41 +0000
1393@@ -6,16 +6,21 @@
1394 __metaclass__ = type
1395
1396 from datetime import datetime
1397+import os.path
1398
1399 import fixtures
1400 import pytz
1401 from testtools.deferredruntest import AsynchronousDeferredRunTest
1402+from testtools.matchers import MatchesListwise
1403 import transaction
1404 from twisted.internet import defer
1405 from twisted.trial.unittest import TestCase as TrialTestCase
1406 from zope.component import getUtility
1407 from zope.security.proxy import Proxy
1408
1409+from lp.archivepublisher.interfaces.archivesigningkey import (
1410+ IArchiveSigningKey,
1411+ )
1412 from lp.buildmaster.enums import BuildStatus
1413 from lp.buildmaster.interfaces.builder import CannotBuild
1414 from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
1415@@ -38,13 +43,17 @@
1416 from lp.soyuz.adapters.archivedependencies import (
1417 get_sources_list_for_building,
1418 )
1419+from lp.soyuz.enums import PackagePublishingStatus
1420 from lp.soyuz.interfaces.archive import ArchiveDisabled
1421 from lp.soyuz.interfaces.livefs import (
1422 LIVEFS_FEATURE_FLAG,
1423 LiveFSBuildArchiveOwnerMismatch,
1424 )
1425 from lp.soyuz.model.livefsbuildbehaviour import LiveFSBuildBehaviour
1426+from lp.soyuz.tests.soyuz import Base64KeyMatches
1427 from lp.testing import TestCaseWithFactory
1428+from lp.testing.gpgkeys import gpgkeysdir
1429+from lp.testing.keyserver import InProcessKeyServerFixture
1430 from lp.testing.layers import LaunchpadZopelessLayer
1431
1432
1433@@ -56,9 +65,13 @@
1434 super(TestLiveFSBuildBehaviourBase, self).setUp()
1435 self.useFixture(FeatureFixture({LIVEFS_FEATURE_FLAG: u"on"}))
1436
1437- def makeJob(self, pocket=PackagePublishingPocket.RELEASE, **kwargs):
1438+ def makeJob(self, archive=None, pocket=PackagePublishingPocket.RELEASE,
1439+ **kwargs):
1440 """Create a sample `ILiveFSBuildBehaviour`."""
1441- distribution = self.factory.makeDistribution(name="distro")
1442+ if archive is None:
1443+ distribution = self.factory.makeDistribution(name="distro")
1444+ else:
1445+ distribution = archive.distribution
1446 distroseries = self.factory.makeDistroSeries(
1447 distribution=distribution, name="unstable")
1448 processor = getUtility(IProcessorSet).getByName("386")
1449@@ -66,7 +79,7 @@
1450 distroseries=distroseries, architecturetag="i386",
1451 processor=processor)
1452 build = self.factory.makeLiveFSBuild(
1453- distroarchseries=distroarchseries, pocket=pocket,
1454+ archive=archive, distroarchseries=distroarchseries, pocket=pocket,
1455 name=u"test-livefs", **kwargs)
1456 return IBuildFarmJobBehaviour(build)
1457
1458@@ -173,8 +186,9 @@
1459 job = self.makeJob(
1460 date_created=datetime(2014, 4, 25, 10, 38, 0, tzinfo=pytz.UTC),
1461 metadata={"project": "distro", "subproject": "special"})
1462- expected_archives = yield get_sources_list_for_building(
1463- job.build, job.build.distro_arch_series, None)
1464+ expected_archives, expected_trusted_keys = (
1465+ yield get_sources_list_for_building(
1466+ job.build, job.build.distro_arch_series, None))
1467 extra_args = yield job._extraBuildArgs()
1468 self.assertEqual({
1469 "archive_private": False,
1470@@ -185,6 +199,7 @@
1471 "project": "distro",
1472 "subproject": "special",
1473 "series": "unstable",
1474+ "trusted_keys": expected_trusted_keys,
1475 }, extra_args)
1476
1477 @defer.inlineCallbacks
1478@@ -209,6 +224,24 @@
1479 self.assertIsNot(Proxy, type(args["lb_args"]))
1480
1481 @defer.inlineCallbacks
1482+ def test_extraBuildArgs_archive_trusted_keys(self):
1483+ # If the archive has a signing key, _extraBuildArgs sends it.
1484+ yield self.useFixture(InProcessKeyServerFixture()).start()
1485+ archive = self.factory.makeArchive()
1486+ key_path = os.path.join(gpgkeysdir, "ppa-sample@canonical.com.sec")
1487+ yield IArchiveSigningKey(archive).setSigningKey(
1488+ key_path, async_keyserver=True)
1489+ job = self.makeJob(archive=archive)
1490+ self.factory.makeBinaryPackagePublishingHistory(
1491+ distroarchseries=job.build.distro_arch_series,
1492+ pocket=job.build.pocket, archive=archive,
1493+ status=PackagePublishingStatus.PUBLISHED)
1494+ args = yield job._extraBuildArgs()
1495+ self.assertThat(args["trusted_keys"], MatchesListwise([
1496+ Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"),
1497+ ]))
1498+
1499+ @defer.inlineCallbacks
1500 def test_composeBuildRequest(self):
1501 job = self.makeJob()
1502 lfa = self.factory.makeLibraryFileAlias(db_only=True)