Merge lp:~cjwatson/launchpad/snap-request-builds-job into lp:launchpad

Proposed by Colin Watson on 2018-06-15
Status: Merged
Merged at revision: 18723
Proposed branch: lp:~cjwatson/launchpad/snap-request-builds-job
Merge into: lp:launchpad
Prerequisite: lp:~cjwatson/launchpad/snap-parse-architectures
Diff against target: 855 lines (+680/-9)
9 files modified
database/schema/security.cfg (+15/-2)
lib/lp/services/config/schema-lazr.conf (+6/-0)
lib/lp/snappy/configure.zcml (+13/-1)
lib/lp/snappy/interfaces/snap.py (+15/-0)
lib/lp/snappy/interfaces/snapjob.py (+97/-0)
lib/lp/snappy/model/snap.py (+47/-6)
lib/lp/snappy/model/snapjob.py (+271/-0)
lib/lp/snappy/tests/test_snap.py (+61/-0)
lib/lp/snappy/tests/test_snapjob.py (+155/-0)
To merge this branch: bzr merge lp:~cjwatson/launchpad/snap-request-builds-job
Reviewer Review Type Date Requested Status
William Grant code 2018-06-15 Approve on 2018-07-11
Review via email: mp+348058@code.launchpad.net

Commit message

Add a job to request builds of a snap for relevant architectures.

Description of the change

See also https://code.launchpad.net/~cjwatson/launchpad/db-snap-job/+merge/348057.

I considered whether to use the existing request-daily-builds DB user or the existing snap-build-job DB user, as I needed a bit of both, or indeed whether to create an entirely new user. In the end I decided it wasn't worth a new user and that snap-build-job was close enough that I could just extend it a bit.

To post a comment you must log in.
William Grant (wgrant) :
review: Approve (code)
18674. By Colin Watson on 2018-07-11

Don't issue a warning when a snap build is already pending.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'database/schema/security.cfg'
2--- database/schema/security.cfg 2018-05-22 06:42:35 +0000
3+++ database/schema/security.cfg 2018-06-15 13:00:33 +0000
4@@ -1,4 +1,4 @@
5-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
6+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
7 # GNU Affero General Public License version 3 (see the file LICENSE).
8 #
9 # Possible permissions: SELECT, INSERT, UPDATE, EXECUTE
10@@ -287,6 +287,7 @@
11 public.snapbuild = SELECT, INSERT, UPDATE, DELETE
12 public.snapbuildjob = SELECT, INSERT, UPDATE, DELETE
13 public.snapfile = SELECT, INSERT, UPDATE, DELETE
14+public.snapjob = SELECT, INSERT, UPDATE, DELETE
15 public.snappydistroseries = SELECT, INSERT, UPDATE, DELETE
16 public.snappyseries = SELECT, INSERT, UPDATE, DELETE
17 public.sourcepackageformatselection = SELECT
18@@ -2537,21 +2538,33 @@
19 groups=script
20 public.account = SELECT
21 public.archive = SELECT
22+public.branch = SELECT
23 public.builder = SELECT
24+public.buildfarmjob = SELECT, INSERT
25+public.buildqueue = SELECT, INSERT, UPDATE
26 public.distribution = SELECT
27 public.distroarchseries = SELECT
28 public.distroseries = SELECT
29 public.emailaddress = SELECT
30+public.gitref = SELECT
31+public.gitrepository = SELECT
32 public.job = SELECT, INSERT, UPDATE
33 public.libraryfilealias = SELECT
34 public.libraryfilecontent = SELECT
35 public.person = SELECT
36 public.personsettings = SELECT
37+public.pocketchroot = SELECT
38+public.processor = SELECT
39+public.product = SELECT
40 public.snap = SELECT, UPDATE
41-public.snapbuild = SELECT, UPDATE
42+public.snaparch = SELECT
43+public.snapbuild = SELECT, INSERT, UPDATE
44 public.snapbuildjob = SELECT, UPDATE
45 public.snapfile = SELECT
46+public.snapjob = SELECT, UPDATE
47 public.snappyseries = SELECT
48+public.sourcepackagename = SELECT
49 public.teammembership = SELECT
50+public.teamparticipation = SELECT
51 public.webhook = SELECT
52 public.webhookjob = SELECT, INSERT
53
54=== modified file 'lib/lp/services/config/schema-lazr.conf'
55--- lib/lp/services/config/schema-lazr.conf 2018-06-15 13:00:33 +0000
56+++ lib/lp/services/config/schema-lazr.conf 2018-06-15 13:00:33 +0000
57@@ -1860,6 +1860,7 @@
58 IRemoveArtifactSubscriptionsJobSource,
59 ISelfRenewalNotificationJobSource,
60 ISevenDayCommercialExpirationJobSource,
61+ ISnapRequestBuildsJobSource,
62 ISnapStoreUploadJobSource,
63 ITeamInvitationNotificationJobSource,
64 ITeamJoinNotificationJobSource,
65@@ -2002,6 +2003,11 @@
66 dbuser: product-job
67 crontab_group: MAIN
68
69+[ISnapRequestBuildsJobSource]
70+module: lp.snappy.interfaces.snapjob
71+dbuser: snap-build-job
72+crontab_group: MAIN
73+
74 [ISnapStoreUploadJobSource]
75 module: lp.snappy.interfaces.snapbuildjob
76 dbuser: snap-build-job
77
78=== modified file 'lib/lp/snappy/configure.zcml'
79--- lib/lp/snappy/configure.zcml 2017-03-20 00:03:52 +0000
80+++ lib/lp/snappy/configure.zcml 2018-06-15 13:00:33 +0000
81@@ -1,4 +1,4 @@
82-<!-- Copyright 2015-2017 Canonical Ltd. This software is licensed under the
83+<!-- Copyright 2015-2018 Canonical Ltd. This software is licensed under the
84 GNU Affero General Public License version 3 (see the file LICENSE).
85 -->
86
87@@ -127,6 +127,18 @@
88 </securedutility>
89
90 <!-- Snap-related jobs -->
91+ <class class="lp.snappy.model.snapjob.SnapJob">
92+ <allow interface="lp.snappy.interfaces.snapjob.ISnapJob" />
93+ </class>
94+ <securedutility
95+ component="lp.snappy.model.snapjob.SnapRequestBuildsJob"
96+ provides="lp.snappy.interfaces.snapjob.ISnapRequestBuildsJobSource">
97+ <allow interface="lp.snappy.interfaces.snapjob.ISnapRequestBuildsJobSource" />
98+ </securedutility>
99+ <class class="lp.snappy.model.snapjob.SnapRequestBuildsJob">
100+ <allow interface="lp.snappy.interfaces.snapjob.ISnapJob" />
101+ <allow interface="lp.snappy.interfaces.snapjob.ISnapRequestBuildsJob" />
102+ </class>
103 <class class="lp.snappy.model.snapbuildjob.SnapBuildJob">
104 <allow interface="lp.snappy.interfaces.snapbuildjob.ISnapBuildJob" />
105 </class>
106
107=== modified file 'lib/lp/snappy/interfaces/snap.py'
108--- lib/lp/snappy/interfaces/snap.py 2018-06-15 13:00:33 +0000
109+++ lib/lp/snappy/interfaces/snap.py 2018-06-15 13:00:33 +0000
110@@ -316,6 +316,21 @@
111 :return: `ISnapBuild`.
112 """
113
114+ def requestBuildsFromJob(requester, archive, pocket, channels=None,
115+ logger=None):
116+ """Synchronous part of `Snap.requestBuilds`.
117+
118+ Request that the snap package be built for relevant architectures.
119+
120+ :param requester: The person requesting the builds.
121+ :param archive: The IArchive to associate the builds with.
122+ :param pocket: The pocket that should be targeted.
123+ :param channels: A dictionary mapping snap names to channels to use
124+ for these builds.
125+ :param logger: An optional logger.
126+ :return: A sequence of `ISnapBuild` instances.
127+ """
128+
129 @operation_parameters(
130 snap_build_ids=List(
131 title=_("A list of snap build ids."),
132
133=== added file 'lib/lp/snappy/interfaces/snapjob.py'
134--- lib/lp/snappy/interfaces/snapjob.py 1970-01-01 00:00:00 +0000
135+++ lib/lp/snappy/interfaces/snapjob.py 2018-06-15 13:00:33 +0000
136@@ -0,0 +1,97 @@
137+# Copyright 2018 Canonical Ltd. This software is licensed under the
138+# GNU Affero General Public License version 3 (see the file LICENSE).
139+
140+"""Snap job interfaces."""
141+
142+from __future__ import absolute_import, print_function, unicode_literals
143+
144+__metaclass__ = type
145+__all__ = [
146+ 'ISnapJob',
147+ 'ISnapRequestBuildsJob',
148+ 'ISnapRequestBuildsJobSource',
149+ ]
150+
151+from lazr.restful.fields import Reference
152+from zope.interface import (
153+ Attribute,
154+ Interface,
155+ )
156+from zope.schema import (
157+ Choice,
158+ Dict,
159+ List,
160+ TextLine,
161+ )
162+
163+from lp import _
164+from lp.registry.interfaces.person import IPerson
165+from lp.registry.interfaces.pocket import PackagePublishingPocket
166+from lp.services.job.interfaces.job import (
167+ IJob,
168+ IJobSource,
169+ IRunnableJob,
170+ )
171+from lp.snappy.interfaces.snap import ISnap
172+from lp.snappy.interfaces.snapbuild import ISnapBuild
173+from lp.soyuz.interfaces.archive import IArchive
174+
175+
176+class ISnapJob(Interface):
177+ """A job related to a snap package."""
178+
179+ job = Reference(
180+ title=_("The common Job attributes."), schema=IJob,
181+ required=True, readonly=True)
182+
183+ snap = Reference(
184+ title=_("The snap package to use for this job."),
185+ schema=ISnap, required=True, readonly=True)
186+
187+ metadata = Attribute(_("A dict of data about the job."))
188+
189+
190+class ISnapRequestBuildsJob(IRunnableJob):
191+ """A Job that processes a request for builds of a snap package."""
192+
193+ requester = Reference(
194+ title=_("The person requesting the builds."), schema=IPerson,
195+ required=True, readonly=True)
196+
197+ archive = Reference(
198+ title=_("The archive to associate the builds with."), schema=IArchive,
199+ required=True, readonly=True)
200+
201+ pocket = Choice(
202+ title=_("The pocket that should be targeted."),
203+ vocabulary=PackagePublishingPocket, required=True, readonly=True)
204+
205+ channels = Dict(
206+ title=_("Source snap channels to use for these builds."),
207+ description=_(
208+ "A dictionary mapping snap names to channels to use for these "
209+ "builds. Currently only 'core' and 'snapcraft' keys are "
210+ "supported."),
211+ key_type=TextLine(), required=False, readonly=True)
212+
213+ error_message = TextLine(
214+ title=_("Error message resulting from running this job."),
215+ required=False, readonly=True)
216+
217+ builds = List(
218+ title=_("The builds created by this request."),
219+ value_type=Reference(schema=ISnapBuild), required=True, readonly=True)
220+
221+
222+class ISnapRequestBuildsJobSource(IJobSource):
223+
224+ def create(snap, requester, archive, pocket, channels):
225+ """Request builds of a snap package.
226+
227+ :param snap: The snap package to build.
228+ :param requester: The person requesting the builds.
229+ :param archive: The IArchive to associate the builds with.
230+ :param pocket: The pocket that should be targeted.
231+ :param channels: A dictionary mapping snap names to channels to use
232+ for these builds.
233+ """
234
235=== modified file 'lib/lp/snappy/model/snap.py'
236--- lib/lp/snappy/model/snap.py 2018-06-15 13:00:33 +0000
237+++ lib/lp/snappy/model/snap.py 2018-06-15 13:00:33 +0000
238@@ -6,10 +6,12 @@
239 'Snap',
240 ]
241
242+from collections import OrderedDict
243 from datetime import (
244 datetime,
245 timedelta,
246 )
247+from operator import attrgetter
248 from urlparse import urlsplit
249
250 from pymacaroons import Macaroon
251@@ -116,6 +118,7 @@
252 from lp.services.webapp.interfaces import ILaunchBag
253 from lp.services.webhooks.interfaces import IWebhookSet
254 from lp.services.webhooks.model import WebhookTargetMixin
255+from lp.snappy.adapters.buildarch import determine_architectures_to_build
256 from lp.snappy.interfaces.snap import (
257 BadSnapSearchContext,
258 BadSnapSource,
259@@ -463,21 +466,25 @@
260 return False
261 return True
262
263- def requestBuild(self, requester, archive, distro_arch_series, pocket,
264- channels=None):
265- """See `ISnap`."""
266+ def _checkRequestBuild(self, requester, archive):
267+ """May `requester` request builds of this snap from `archive`?"""
268 if not requester.inTeam(self.owner):
269 raise SnapNotOwner(
270 "%s cannot create snap package builds owned by %s." %
271 (requester.displayname, self.owner.displayname))
272 if not archive.enabled:
273 raise ArchiveDisabled(archive.displayname)
274- if distro_arch_series not in self.getAllowedArchitectures():
275- raise SnapBuildDisallowedArchitecture(distro_arch_series)
276 if archive.private and self.owner != archive.owner:
277 # See rationale in `SnapBuildArchiveOwnerMismatch` docstring.
278 raise SnapBuildArchiveOwnerMismatch()
279
280+ def requestBuild(self, requester, archive, distro_arch_series, pocket,
281+ channels=None):
282+ """See `ISnap`."""
283+ self._checkRequestBuild(requester, archive)
284+ if distro_arch_series not in self.getAllowedArchitectures():
285+ raise SnapBuildDisallowedArchitecture(distro_arch_series)
286+
287 pending = IStore(self).find(
288 SnapBuild,
289 SnapBuild.snap_id == self.id,
290@@ -495,8 +502,42 @@
291 build.queueBuild()
292 return build
293
294+ def requestBuildsFromJob(self, requester, archive, pocket, channels=None,
295+ logger=None):
296+ """See `ISnap`."""
297+ snapcraft_data = removeSecurityProxy(
298+ getUtility(ISnapSet).getSnapcraftYaml(self))
299+ # Sort by Processor.id for determinism. This is chosen to be the
300+ # same order as in BinaryPackageBuildSet.createForSource, to
301+ # minimise confusion.
302+ supported_arches = OrderedDict(
303+ (das.architecturetag, das) for das in sorted(
304+ self.getAllowedArchitectures(),
305+ key=attrgetter("processor.id")))
306+ architectures_to_build = determine_architectures_to_build(
307+ snapcraft_data, supported_arches.keys())
308+
309+ builds = []
310+ for build_instance in architectures_to_build:
311+ arch = build_instance.architecture
312+ try:
313+ build = self.requestBuild(
314+ requester, archive, supported_arches[arch], pocket,
315+ channels)
316+ if logger is not None:
317+ logger.debug(
318+ " - %s/%s/%s: Build requested.",
319+ self.owner.name, self.name, arch)
320+ builds.append(build)
321+ except SnapBuildAlreadyPending as e:
322+ if logger is not None:
323+ logger.warning(
324+ " - %s/%s/%s: %s",
325+ self.owner.name, self.name, arch, e)
326+ return builds
327+
328 def requestAutoBuilds(self, allow_failures=False, logger=None):
329- """See `ISnapSet`."""
330+ """See `ISnap`."""
331 builds = []
332 if self.auto_build_archive is None:
333 raise CannotRequestAutoBuilds("auto_build_archive")
334
335=== added file 'lib/lp/snappy/model/snapjob.py'
336--- lib/lp/snappy/model/snapjob.py 1970-01-01 00:00:00 +0000
337+++ lib/lp/snappy/model/snapjob.py 2018-06-15 13:00:33 +0000
338@@ -0,0 +1,271 @@
339+# Copyright 2018 Canonical Ltd. This software is licensed under the
340+# GNU Affero General Public License version 3 (see the file LICENSE).
341+
342+"""Snap package jobs."""
343+
344+from __future__ import absolute_import, print_function, unicode_literals
345+
346+__metaclass__ = type
347+__all__ = [
348+ 'SnapJob',
349+ 'SnapJobType',
350+ 'SnapRequestBuildsJob',
351+ ]
352+
353+from lazr.delegates import delegate_to
354+from lazr.enum import (
355+ DBEnumeratedType,
356+ DBItem,
357+ )
358+from storm.locals import (
359+ Int,
360+ JSON,
361+ Reference,
362+ )
363+from storm.store import EmptyResultSet
364+import transaction
365+from zope.component import getUtility
366+from zope.interface import (
367+ implementer,
368+ provider,
369+ )
370+
371+from lp.app.errors import NotFoundError
372+from lp.registry.interfaces.person import IPersonSet
373+from lp.registry.interfaces.pocket import PackagePublishingPocket
374+from lp.services.config import config
375+from lp.services.database.enumcol import EnumCol
376+from lp.services.database.interfaces import (
377+ IMasterStore,
378+ IStore,
379+ )
380+from lp.services.database.stormbase import StormBase
381+from lp.services.job.model.job import (
382+ EnumeratedSubclass,
383+ Job,
384+ )
385+from lp.services.job.runner import BaseRunnableJob
386+from lp.services.mail.sendmail import format_address_for_person
387+from lp.services.propertycache import cachedproperty
388+from lp.services.scripts import log
389+from lp.snappy.interfaces.snap import (
390+ CannotFetchSnapcraftYaml,
391+ CannotParseSnapcraftYaml,
392+ )
393+from lp.snappy.interfaces.snapjob import (
394+ ISnapJob,
395+ ISnapRequestBuildsJob,
396+ ISnapRequestBuildsJobSource,
397+ )
398+from lp.snappy.model.snapbuild import SnapBuild
399+from lp.soyuz.model.archive import Archive
400+
401+
402+class SnapJobType(DBEnumeratedType):
403+ """Values that `ISnapJob.job_type` can take."""
404+
405+ REQUEST_BUILDS = DBItem(0, """
406+ Request builds
407+
408+ This job requests builds of a snap package.
409+ """)
410+
411+
412+@implementer(ISnapJob)
413+class SnapJob(StormBase):
414+ """See `ISnapJob`."""
415+
416+ __storm_table__ = 'SnapJob'
417+
418+ job_id = Int(name='job', primary=True, allow_none=False)
419+ job = Reference(job_id, 'Job.id')
420+
421+ snap_id = Int(name='snap', allow_none=False)
422+ snap = Reference(snap_id, 'Snap.id')
423+
424+ job_type = EnumCol(enum=SnapJobType, notNull=True)
425+
426+ metadata = JSON('json_data', allow_none=False)
427+
428+ def __init__(self, snap, job_type, metadata, **job_args):
429+ """Constructor.
430+
431+ Extra keyword arguments are used to construct the underlying Job
432+ object.
433+
434+ :param snap: The `ISnap` this job relates to.
435+ :param job_type: The `SnapJobType` of this job.
436+ :param metadata: The type-specific variables, as a JSON-compatible
437+ dict.
438+ """
439+ super(SnapJob, self).__init__()
440+ self.job = Job(**job_args)
441+ self.snap = snap
442+ self.job_type = job_type
443+ self.metadata = metadata
444+
445+ def makeDerived(self):
446+ return SnapJobDerived.makeSubclass(self)
447+
448+
449+@delegate_to(ISnapJob)
450+class SnapJobDerived(BaseRunnableJob):
451+
452+ __metaclass__ = EnumeratedSubclass
453+
454+ def __init__(self, snap_job):
455+ self.context = snap_job
456+
457+ def __repr__(self):
458+ """An informative representation of the job."""
459+ return "<%s for ~%s/+snap/%s>" % (
460+ self.__class__.__name__, self.snap.owner.name, self.snap.name)
461+
462+ @classmethod
463+ def get(cls, job_id):
464+ """Get a job by id.
465+
466+ :return: The `SnapJob` with the specified id, as the current
467+ `SnapJobDerived` subclass.
468+ :raises: `NotFoundError` if there is no job with the specified id,
469+ or its `job_type` does not match the desired subclass.
470+ """
471+ snap_job = IStore(SnapJob).get(SnapJob, job_id)
472+ if snap_job.job_type != cls.class_job_type:
473+ raise NotFoundError(
474+ "No object found with id %d and type %s" %
475+ (job_id, cls.class_job_type.title))
476+ return cls(snap_job)
477+
478+ @classmethod
479+ def iterReady(cls):
480+ """See `IJobSource`."""
481+ jobs = IMasterStore(SnapJob).find(
482+ SnapJob,
483+ SnapJob.job_type == cls.class_job_type,
484+ SnapJob.job == Job.id,
485+ Job.id.is_in(Job.ready_jobs))
486+ return (cls(job) for job in jobs)
487+
488+ def getOopsVars(self):
489+ """See `IRunnableJob`."""
490+ oops_vars = super(SnapJobDerived, self).getOopsVars()
491+ oops_vars.extend([
492+ ("job_id", self.context.job.id),
493+ ("job_type", self.context.job_type.title),
494+ ("snap_owner_name", self.context.snap.owner.name),
495+ ("snap_name", self.context.snap.name),
496+ ])
497+ return oops_vars
498+
499+
500+@implementer(ISnapRequestBuildsJob)
501+@provider(ISnapRequestBuildsJobSource)
502+class SnapRequestBuildsJob(SnapJobDerived):
503+ """A Job that processes a request for builds of a snap package."""
504+
505+ class_job_type = SnapJobType.REQUEST_BUILDS
506+
507+ user_error_types = (CannotParseSnapcraftYaml, NotFoundError)
508+ retry_error_types = (CannotFetchSnapcraftYaml,)
509+
510+ max_retries = 5
511+
512+ config = config.ISnapRequestBuildsJobSource
513+
514+ @classmethod
515+ def create(cls, snap, requester, archive, pocket, channels):
516+ """See `ISnapRequestBuildsJobSource`."""
517+ metadata = {
518+ "requester": requester.id,
519+ "archive": archive.id,
520+ "pocket": pocket.value,
521+ "channels": channels,
522+ }
523+ snap_job = SnapJob(snap, cls.class_job_type, metadata)
524+ job = cls(snap_job)
525+ job.celeryRunOnCommit()
526+ return job
527+
528+ def getOperationDescription(self):
529+ return "requesting builds of %s" % self.snap.name
530+
531+ def getErrorRecipients(self):
532+ if self.requester is None or self.requester.preferredemail is None:
533+ return []
534+ return [format_address_for_person(self.requester)]
535+
536+ @cachedproperty
537+ def requester(self):
538+ """See `ISnapRequestBuildsJob`."""
539+ requester_id = self.metadata["requester"]
540+ return getUtility(IPersonSet).get(requester_id)
541+
542+ @cachedproperty
543+ def archive(self):
544+ """See `ISnapRequestBuildsJob`."""
545+ archive_id = self.metadata["archive"]
546+ return IStore(Archive).find(Archive, Archive.id == archive_id).one()
547+
548+ @property
549+ def pocket(self):
550+ """See `ISnapRequestBuildsJob`."""
551+ name = self.metadata["pocket"]
552+ return PackagePublishingPocket.items[name]
553+
554+ @property
555+ def channels(self):
556+ """See `ISnapRequestBuildsJob`."""
557+ return self.metadata["channels"]
558+
559+ @property
560+ def error_message(self):
561+ """See `ISnapRequestBuildsJob`."""
562+ return self.metadata.get("error_message")
563+
564+ @error_message.setter
565+ def error_message(self, message):
566+ """See `ISnapRequestBuildsJob`."""
567+ self.metadata["error_message"] = message
568+
569+ @property
570+ def builds(self):
571+ """See `ISnapRequestBuildsJob`."""
572+ build_ids = self.metadata.get("builds")
573+ if build_ids is None:
574+ return EmptyResultSet()
575+ else:
576+ return IStore(SnapBuild).find(
577+ SnapBuild, SnapBuild.id.is_in(build_ids))
578+
579+ @builds.setter
580+ def builds(self, builds):
581+ """See `ISnapRequestBuildsJob`."""
582+ self.metadata["builds"] = [build.id for build in builds]
583+
584+ def run(self):
585+ """See `IRunnableJob`."""
586+ requester = self.requester
587+ if requester is None:
588+ log.info(
589+ "Skipping %r because the requester has been deleted." % self)
590+ return
591+ archive = self.archive
592+ if archive is None:
593+ log.info(
594+ "Skipping %r because the archive has been deleted." % self)
595+ return
596+ try:
597+ self.builds = self.snap.requestBuildsFromJob(
598+ requester, archive, self.pocket, channels=self.channels,
599+ logger=log)
600+ self.error_message = None
601+ except self.retry_error_types:
602+ raise
603+ except Exception as e:
604+ self.error_message = str(e)
605+ # The normal job infrastructure will abort the transaction, but
606+ # we want to commit instead: the only database changes we make
607+ # are to this job's metadata and should be preserved.
608+ transaction.commit()
609+ raise
610
611=== modified file 'lib/lp/snappy/tests/test_snap.py'
612--- lib/lp/snappy/tests/test_snap.py 2018-06-15 13:00:33 +0000
613+++ lib/lp/snappy/tests/test_snap.py 2018-06-15 13:00:33 +0000
614@@ -12,6 +12,7 @@
615 timedelta,
616 )
617 import json
618+from textwrap import dedent
619 from urlparse import urlsplit
620
621 from fixtures import MockPatch
622@@ -91,6 +92,7 @@
623 ISnapBuildSet,
624 )
625 from lp.snappy.interfaces.snapbuildjob import ISnapStoreUploadJobSource
626+from lp.snappy.interfaces.snapjob import ISnapRequestBuildsJobSource
627 from lp.snappy.interfaces.snapstoreclient import ISnapStoreClient
628 from lp.snappy.model.snap import SnapSet
629 from lp.snappy.model.snapbuild import SnapFile
630@@ -361,6 +363,65 @@
631 snap.owner, snap.distro_series.main_archive, distroarchseries,
632 PackagePublishingPocket.UPDATES)
633
634+ def makeRequestBuildsJob(self, arch_tags):
635+ distro = self.factory.makeDistribution()
636+ distroseries = self.factory.makeDistroSeries(distribution=distro)
637+ processors = [
638+ self.factory.makeProcessor(
639+ name=arch_tag, supports_virtualized=True)
640+ for arch_tag in arch_tags]
641+ for processor in processors:
642+ das = self.factory.makeDistroArchSeries(
643+ distroseries=distroseries, architecturetag=processor.name,
644+ processor=processor)
645+ das.addOrUpdateChroot(self.factory.makeLibraryFileAlias(
646+ filename="fake_chroot.tar.gz", db_only=True))
647+ [git_ref] = self.factory.makeGitRefs()
648+ snap = self.factory.makeSnap(
649+ git_ref=git_ref, distroseries=distroseries, processors=processors)
650+ return getUtility(ISnapRequestBuildsJobSource).create(
651+ snap, snap.owner.teamowner, distro.main_archive,
652+ PackagePublishingPocket.RELEASE, {"snapcraft": "edge"})
653+
654+ def assertRequestedBuildsMatch(self, builds, job, arch_tags):
655+ self.assertThat(builds, MatchesSetwise(
656+ *(MatchesStructure(
657+ requester=Equals(job.requester),
658+ snap=Equals(job.snap),
659+ archive=Equals(job.archive),
660+ distro_arch_series=Equals(job.snap.distro_series[arch_tag]),
661+ pocket=Equals(job.pocket),
662+ channels=Equals(job.channels))
663+ for arch_tag in arch_tags)))
664+
665+ def test_requestBuildsFromJob_restricts_explicit_list(self):
666+ # requestBuildsFromJob limits builds targeted at an explicit list of
667+ # architectures to those allowed for the snap.
668+ self.useFixture(GitHostingFixture(blob=dedent("""\
669+ architectures:
670+ - build-on: sparc
671+ - build-on: i386
672+ - build-on: avr
673+ """)))
674+ job = self.makeRequestBuildsJob(["sparc"])
675+ with person_logged_in(job.requester):
676+ builds = job.snap.requestBuildsFromJob(
677+ job.requester, job.archive, job.pocket,
678+ removeSecurityProxy(job.channels))
679+ self.assertRequestedBuildsMatch(builds, job, ["sparc"])
680+
681+ def test_requestBuildsFromJob_no_explicit_architectures(self):
682+ # If the snap doesn't specify any architectures,
683+ # requestBuildsFromJob requests builds for all configured
684+ # architectures.
685+ self.useFixture(GitHostingFixture(blob="name: foo\n"))
686+ job = self.makeRequestBuildsJob(["mips64el", "riscv64"])
687+ with person_logged_in(job.requester):
688+ builds = job.snap.requestBuildsFromJob(
689+ job.requester, job.archive, job.pocket,
690+ removeSecurityProxy(job.channels))
691+ self.assertRequestedBuildsMatch(builds, job, ["mips64el", "riscv64"])
692+
693 def test_requestAutoBuilds(self):
694 # requestAutoBuilds creates new builds for all configured
695 # architectures with appropriate parameters.
696
697=== added file 'lib/lp/snappy/tests/test_snapjob.py'
698--- lib/lp/snappy/tests/test_snapjob.py 1970-01-01 00:00:00 +0000
699+++ lib/lp/snappy/tests/test_snapjob.py 2018-06-15 13:00:33 +0000
700@@ -0,0 +1,155 @@
701+# Copyright 2018 Canonical Ltd. This software is licensed under the
702+# GNU Affero General Public License version 3 (see the file LICENSE).
703+
704+"""Tests for snap package jobs."""
705+
706+from __future__ import absolute_import, print_function, unicode_literals
707+
708+__metaclass__ = type
709+
710+from textwrap import dedent
711+
712+from testtools.matchers import (
713+ AfterPreprocessing,
714+ ContainsDict,
715+ Equals,
716+ Is,
717+ MatchesSetwise,
718+ MatchesStructure,
719+ )
720+
721+from lp.code.tests.helpers import GitHostingFixture
722+from lp.registry.interfaces.pocket import PackagePublishingPocket
723+from lp.services.config import config
724+from lp.services.job.interfaces.job import JobStatus
725+from lp.services.job.runner import JobRunner
726+from lp.services.mail.sendmail import format_address_for_person
727+from lp.snappy.interfaces.snap import CannotParseSnapcraftYaml
728+from lp.snappy.interfaces.snapjob import (
729+ ISnapJob,
730+ ISnapRequestBuildsJob,
731+ )
732+from lp.snappy.model.snapjob import (
733+ SnapJob,
734+ SnapJobType,
735+ SnapRequestBuildsJob,
736+ )
737+from lp.testing import TestCaseWithFactory
738+from lp.testing.dbuser import dbuser
739+from lp.testing.layers import ZopelessDatabaseLayer
740+
741+
742+class TestSnapJob(TestCaseWithFactory):
743+
744+ layer = ZopelessDatabaseLayer
745+
746+ def test_provides_interface(self):
747+ # `SnapJob` objects provide `ISnapJob`.
748+ snap = self.factory.makeSnap()
749+ self.assertProvides(
750+ SnapJob(snap, SnapJobType.REQUEST_BUILDS, {}), ISnapJob)
751+
752+
753+class TestSnapRequestBuildsJob(TestCaseWithFactory):
754+
755+ layer = ZopelessDatabaseLayer
756+
757+ def test_provides_interface(self):
758+ # `SnapRequestBuildsJob` objects provide `ISnapRequestBuildsJob`."""
759+ snap = self.factory.makeSnap()
760+ archive = self.factory.makeArchive()
761+ job = SnapRequestBuildsJob.create(
762+ snap, snap.registrant, archive, PackagePublishingPocket.RELEASE,
763+ None)
764+ self.assertProvides(job, ISnapRequestBuildsJob)
765+
766+ def test___repr__(self):
767+ # `SnapRequestBuildsJob` objects have an informative __repr__.
768+ snap = self.factory.makeSnap()
769+ archive = self.factory.makeArchive()
770+ job = SnapRequestBuildsJob.create(
771+ snap, snap.registrant, archive, PackagePublishingPocket.RELEASE,
772+ None)
773+ self.assertEqual(
774+ "<SnapRequestBuildsJob for ~%s/+snap/%s>" % (
775+ snap.owner.name, snap.name),
776+ repr(job))
777+
778+ def makeSeriesAndProcessors(self, arch_tags):
779+ distro = self.factory.makeDistribution()
780+ distroseries = self.factory.makeDistroSeries(distribution=distro)
781+ processors = [
782+ self.factory.makeProcessor(
783+ name=arch_tag, supports_virtualized=True)
784+ for arch_tag in arch_tags]
785+ for processor in processors:
786+ das = self.factory.makeDistroArchSeries(
787+ distroseries=distroseries, architecturetag=processor.name,
788+ processor=processor)
789+ das.addOrUpdateChroot(self.factory.makeLibraryFileAlias(
790+ filename="fake_chroot.tar.gz", db_only=True))
791+ return distroseries, processors
792+
793+ def test_run(self):
794+ # The job requests builds and records the result.
795+ distroseries, processors = self.makeSeriesAndProcessors(
796+ ["avr2001", "sparc64", "x32"])
797+ [git_ref] = self.factory.makeGitRefs()
798+ snap = self.factory.makeSnap(
799+ git_ref=git_ref, distroseries=distroseries, processors=processors)
800+ job = SnapRequestBuildsJob.create(
801+ snap, snap.registrant, distroseries.main_archive,
802+ PackagePublishingPocket.RELEASE, {"core": "stable"})
803+ snapcraft_yaml = dedent("""\
804+ architectures:
805+ - build-on: avr2001
806+ - build-on: x32
807+ """)
808+ self.useFixture(GitHostingFixture(blob=snapcraft_yaml))
809+ with dbuser(config.ISnapRequestBuildsJobSource.dbuser):
810+ JobRunner([job]).runAll()
811+ self.assertEmailQueueLength(0)
812+ self.assertThat(job, MatchesStructure(
813+ job=MatchesStructure.byEquality(status=JobStatus.COMPLETED),
814+ error_message=Is(None),
815+ builds=AfterPreprocessing(set, MatchesSetwise(*[
816+ MatchesStructure.byEquality(
817+ requester=snap.registrant,
818+ snap=snap,
819+ archive=distroseries.main_archive,
820+ distro_arch_series=distroseries[arch],
821+ pocket=PackagePublishingPocket.RELEASE,
822+ channels={"core": "stable"})
823+ for arch in ("avr2001", "x32")]))))
824+
825+ def test_run_failed(self):
826+ # A failed run sets the job status to FAILED and records the error
827+ # message.
828+ # The job requests builds and records the result.
829+ distroseries, processors = self.makeSeriesAndProcessors(
830+ ["avr2001", "sparc64", "x32"])
831+ [git_ref] = self.factory.makeGitRefs()
832+ snap = self.factory.makeSnap(
833+ git_ref=git_ref, distroseries=distroseries, processors=processors)
834+ job = SnapRequestBuildsJob.create(
835+ snap, snap.registrant, distroseries.main_archive,
836+ PackagePublishingPocket.RELEASE, {"core": "stable"})
837+ self.useFixture(GitHostingFixture()).getBlob.failure = (
838+ CannotParseSnapcraftYaml("Nonsense on stilts"))
839+ with dbuser(config.ISnapRequestBuildsJobSource.dbuser):
840+ JobRunner([job]).runAll()
841+ [notification] = self.assertEmailQueueLength(1)
842+ self.assertThat(dict(notification), ContainsDict({
843+ "From": Equals(config.canonical.noreply_from_address),
844+ "To": Equals(format_address_for_person(snap.registrant)),
845+ "Subject": Equals(
846+ "Launchpad error while requesting builds of %s" % snap.name),
847+ }))
848+ self.assertEqual(
849+ "Launchpad encountered an error during the following operation: "
850+ "requesting builds of %s. Nonsense on stilts" % snap.name,
851+ notification.get_payload(decode=True))
852+ self.assertThat(job, MatchesStructure(
853+ job=MatchesStructure.byEquality(status=JobStatus.FAILED),
854+ error_message=Equals("Nonsense on stilts"),
855+ builds=AfterPreprocessing(set, MatchesSetwise())))