launchpad-buildd

Merge lp:~cjwatson/launchpad-buildd/gatherResults-in-thread into lp:launchpad-buildd

  1. gatherResults-in-thread
  2. Merge into trunk
Proposed by Colin Watson
Status: Merged
Merged at revision: 350
Proposed branch: lp:~cjwatson/launchpad-buildd/gatherResults-in-thread
Merge into: lp:launchpad-buildd
Diff against target: 930 lines (+161/-75)
12 files modified
debian/changelog (+2/-0)
lpbuildd/binarypackage.py (+2/-9)
lpbuildd/debian.py (+37/-2)
lpbuildd/livefs.py (+2/-2)
lpbuildd/snap.py (+2/-2)
lpbuildd/sourcepackagerecipe.py (+4/-2)
lpbuildd/tests/test_binarypackage.py (+52/-28)
lpbuildd/tests/test_livefs.py (+14/-6)
lpbuildd/tests/test_snap.py (+12/-8)
lpbuildd/tests/test_sourcepackagerecipe.py (+19/-9)
lpbuildd/tests/test_translationtemplatesbuildmanager.py (+13/-5)
lpbuildd/translationtemplates.py (+2/-2)
To merge this branch: bzr merge lp:~cjwatson/launchpad-buildd/gatherResults-in-thread
Reviewer Review Type Date Requested Status
William Grant code Approve
Review via email: mp+356196@code.launchpad.net

Commit message

Call gatherResults in a different thread so that it doesn't block responses to XML-RPC requests.

Description of the change

The test changes here really just keep the existing tests working. I tried to get proper tests going for this, but I couldn't quite manage to hook up all the callbacks in a way that didn't crash or hang, and ran out of time for it. However, I've done a fair amount of local testing of these changes, including manually inserting time.sleep calls into gatherResults for various build types and testing that I can usefully call status() and abort() while gatherResults is in progress.

To post a comment you must log in.
Revision history for this message
William Grant (wgrant) wrote :

Would it make sense to factor the gatherResults wrapper out into a deferGatherResults or similar? There's also no particular reason that the other four job types couldn't have the "Failed to gather results" handling too.

review: Approve (code)
lp:~cjwatson/launchpad-buildd/gatherResults-in-thread updated
349. By Colin Watson

Factor out common gatherResults thread handling.

350. By Colin Watson

Merge trunk.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'debian/changelog'
2--- debian/changelog 2018-10-11 08:59:22 +0000
3+++ debian/changelog 2018-10-19 06:46:37 +0000
4@@ -8,6 +8,8 @@
5
6 [ Colin Watson ]
7 * Set SNAPCRAFT_BUILD_ENVIRONMENT=host when building snaps (LP: #1791201).
8+ * Call gatherResults in a different thread so that it doesn't block
9+ responses to XML-RPC requests (LP: #1795877).
10
11 -- Colin Watson <cjwatson@ubuntu.com> Tue, 09 Oct 2018 11:38:07 +0200
12
13
14=== modified file 'lpbuildd/binarypackage.py'
15--- lpbuildd/binarypackage.py 2017-08-03 13:13:08 +0000
16+++ lpbuildd/binarypackage.py 2018-10-19 06:46:37 +0000
17@@ -1,4 +1,4 @@
18-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
19+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
20 # GNU Affero General Public License version 3 (see the file LICENSE).
21
22 from __future__ import absolute_import, print_function
23@@ -343,14 +343,7 @@
24 """Finished the sbuild run."""
25 if success == SBuildExitCodes.OK:
26 print("Returning build status: OK")
27- try:
28- self.gatherResults()
29- except Exception as e:
30- self._slave.log("Failed to gather results: %s" % e)
31- self._slave.buildFail()
32- self.alreadyfailed = True
33- self.doReapProcesses(self._state)
34- return
35+ return self.deferGatherResults()
36
37 log_patterns = []
38 stop_patterns = [["^Toolchain package versions:", re.M]]
39
40=== modified file 'lpbuildd/debian.py'
41--- lpbuildd/debian.py 2018-05-02 09:00:10 +0000
42+++ lpbuildd/debian.py 2018-10-19 06:46:37 +0000
43@@ -1,4 +1,4 @@
44-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
45+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
46 # GNU Affero General Public License version 3 (see the file LICENSE).
47
48 # Authors: Daniel Silverstone <daniel.silverstone@canonical.com>
49@@ -13,6 +13,10 @@
50 import re
51 import signal
52
53+from twisted.internet import (
54+ defer,
55+ threads,
56+ )
57 from twisted.python import log
58
59 from lpbuildd.slave import (
60@@ -41,6 +45,7 @@
61 self._state = DebianBuildState.INIT
62 slave.emptyLog()
63 self.alreadyfailed = False
64+ self._iterator = None
65
66 @property
67 def initial_build_state(self):
68@@ -115,6 +120,27 @@
69 finally:
70 chfile.close()
71
72+ def deferGatherResults(self):
73+ """Gather the results of the build in a thread."""
74+ # XXX cjwatson 2018-10-04: Refactor using inlineCallbacks once we're
75+ # on Twisted >= 18.7.0 (https://twistedmatrix.com/trac/ticket/4632).
76+ def failed_to_gather(failure):
77+ if failure.check(defer.CancelledError):
78+ if not self.alreadyfailed:
79+ self._slave.log("Build cancelled unexpectedly!")
80+ self._slave.buildFail()
81+ else:
82+ self._slave.log("Failed to gather results: %s" % failure.value)
83+ self._slave.buildFail()
84+ self.alreadyfailed = True
85+
86+ def reap(ignored):
87+ self.doReapProcesses(self._state)
88+
89+ return threads.deferToThread(self.gatherResults).addErrback(
90+ failed_to_gather).addCallback(reap)
91+
92+ @defer.inlineCallbacks
93 def iterate(self, success, quiet=False):
94 # When a Twisted ProcessControl class is killed by SIGTERM,
95 # which we call 'build process aborted', 'None' is returned as
96@@ -129,7 +155,9 @@
97 func = getattr(self, "iterate_" + self._state, None)
98 if func is None:
99 raise ValueError("Unknown internal state " + self._state)
100- func(success)
101+ self._iterator = func(success)
102+ yield self._iterator
103+ self._iterator = None
104
105 def iterateReap(self, state, success):
106 log.msg("Iterating with success flag %s against stage %s after "
107@@ -305,6 +333,13 @@
108 """
109 self.doReapProcesses(self._state, notify=False)
110
111+ def abort(self):
112+ """See `BuildManager`."""
113+ super(DebianBuildManager, self).abort()
114+ if self._iterator is not None:
115+ self._iterator.cancel()
116+ self._iterator = None
117+
118
119 def get_build_path(home, build_id, *extra):
120 """Generate a path within the build directory.
121
122=== modified file 'lpbuildd/livefs.py'
123--- lpbuildd/livefs.py 2018-10-09 09:46:07 +0000
124+++ lpbuildd/livefs.py 2018-10-19 06:46:37 +0000
125@@ -1,4 +1,4 @@
126-# Copyright 2013-2017 Canonical Ltd. This software is licensed under the
127+# Copyright 2013-2018 Canonical Ltd. This software is licensed under the
128 # GNU Affero General Public License version 3 (see the file LICENSE).
129
130 __metaclass__ = type
131@@ -72,8 +72,8 @@
132 def iterate_BUILD_LIVEFS(self, retcode):
133 """Finished building the live filesystem."""
134 if retcode == RETCODE_SUCCESS:
135- self.gatherResults()
136 print("Returning build status: OK")
137+ return self.deferGatherResults()
138 elif (retcode >= RETCODE_FAILURE_INSTALL and
139 retcode <= RETCODE_FAILURE_BUILD):
140 if not self.alreadyfailed:
141
142=== modified file 'lpbuildd/snap.py'
143--- lpbuildd/snap.py 2018-06-07 20:08:26 +0000
144+++ lpbuildd/snap.py 2018-10-19 06:46:37 +0000
145@@ -1,4 +1,4 @@
146-# Copyright 2015-2016 Canonical Ltd. This software is licensed under the
147+# Copyright 2015-2018 Canonical Ltd. This software is licensed under the
148 # GNU Affero General Public License version 3 (see the file LICENSE).
149
150 from __future__ import print_function
151@@ -356,8 +356,8 @@
152 self.stopProxy()
153 self.revokeProxyToken()
154 if retcode == RETCODE_SUCCESS:
155- self.gatherResults()
156 print("Returning build status: OK")
157+ return self.deferGatherResults()
158 elif (retcode >= RETCODE_FAILURE_INSTALL and
159 retcode <= RETCODE_FAILURE_BUILD):
160 if not self.alreadyfailed:
161
162=== modified file 'lpbuildd/sourcepackagerecipe.py'
163--- lpbuildd/sourcepackagerecipe.py 2016-01-05 14:05:42 +0000
164+++ lpbuildd/sourcepackagerecipe.py 2018-10-19 06:46:37 +0000
165@@ -1,4 +1,4 @@
166-# Copyright 2010 Canonical Ltd. This software is licensed under the
167+# Copyright 2010-2018 Canonical Ltd. This software is licensed under the
168 # GNU Affero General Public License version 3 (see the file LICENSE).
169 # pylint: disable-msg=E1002
170
171@@ -12,6 +12,8 @@
172 DebianBuildState,
173 get_build_path,
174 )
175+
176+
177 RETCODE_SUCCESS = 0
178 RETCODE_FAILURE_INSTALL = 200
179 RETCODE_FAILURE_BUILD_TREE = 201
180@@ -98,8 +100,8 @@
181 def iterate_BUILD_RECIPE(self, retcode):
182 """Move from BUILD_RECIPE to the next logical state."""
183 if retcode == RETCODE_SUCCESS:
184- self.gatherResults()
185 print("Returning build status: OK")
186+ return self.deferGatherResults()
187 elif retcode == RETCODE_FAILURE_INSTALL_BUILD_DEPS:
188 if not self.alreadyfailed:
189 rx = (
190
191=== modified file 'lpbuildd/tests/test_binarypackage.py'
192--- lpbuildd/tests/test_binarypackage.py 2017-08-22 14:53:29 +0000
193+++ lpbuildd/tests/test_binarypackage.py 2018-10-19 06:46:37 +0000
194@@ -1,4 +1,4 @@
195-# Copyright 2013-2017 Canonical Ltd. This software is licensed under the
196+# Copyright 2013-2018 Canonical Ltd. This software is licensed under the
197 # GNU Affero General Public License version 3 (see the file LICENSE).
198
199 __metaclass__ = type
200@@ -13,6 +13,7 @@
201 from debian.deb822 import PkgRelation
202 from fixtures import MonkeyPatch
203 from testtools import TestCase
204+from testtools.deferredruntest import AsynchronousDeferredRunTest
205 from testtools.matchers import (
206 Contains,
207 ContainsDict,
208@@ -21,6 +22,7 @@
209 MatchesListwise,
210 Not,
211 )
212+from twisted.internet import defer
213 from twisted.internet.task import Clock
214
215 from lpbuildd.binarypackage import (
216@@ -85,6 +87,9 @@
217
218 class TestBinaryPackageBuildManagerIteration(TestCase):
219 """Run BinaryPackageBuildManager through its iteration steps."""
220+
221+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
222+
223 def setUp(self):
224 super(TestBinaryPackageBuildManagerIteration, self).setUp()
225 self.useFixture(DisableSudo())
226@@ -108,6 +113,7 @@
227 """Retrieve build manager's state."""
228 return self.buildmanager._state
229
230+ @defer.inlineCallbacks
231 def startBuild(self, dscname=''):
232 # The build manager's iterate() kicks off the consecutive states
233 # after INIT.
234@@ -123,7 +129,7 @@
235 self.buildmanager._state = BinaryPackageBuildState.UPDATE
236
237 # SBUILD: Build the package.
238- self.buildmanager.iterate(0)
239+ yield self.buildmanager.iterate(0)
240 self.assertState(
241 BinaryPackageBuildState.SBUILD,
242 [
243@@ -147,9 +153,10 @@
244 self.assertNotEqual(
245 self.buildmanager.iterate, self.buildmanager.iterators[-1])
246
247+ @defer.inlineCallbacks
248 def assertScansSanely(self, exit_code):
249 # After building the package, reap processes.
250- self.buildmanager.iterate(exit_code)
251+ yield self.buildmanager.iterate(exit_code)
252 self.assertState(
253 BinaryPackageBuildState.SBUILD,
254 ['sharepath/slavebin/in-target', 'in-target',
255@@ -166,9 +173,10 @@
256 '--backend=chroot', '--series=warty', '--arch=i386',
257 self.buildid], final=True)
258
259+ @defer.inlineCallbacks
260 def test_iterate(self):
261 # The build manager iterates a normal build from start to finish.
262- self.startBuild()
263+ yield self.startBuild()
264
265 write_file(
266 os.path.join(self.buildmanager._cachepath, 'buildlog'),
267@@ -179,7 +187,7 @@
268 write_file(changes_path, "I am a changes file.")
269
270 # After building the package, reap processes.
271- self.assertScansSanely(SBuildExitCodes.OK)
272+ yield self.assertScansSanely(SBuildExitCodes.OK)
273 self.assertFalse(self.slave.wasCalled('buildFail'))
274 self.assertThat(self.slave, HasWaitingFiles.byEquality({
275 'foo_1_i386.changes': b'I am a changes file.',
276@@ -189,6 +197,7 @@
277 self.assertUnmountsSanely()
278 self.assertFalse(self.slave.wasCalled('buildFail'))
279
280+ @defer.inlineCallbacks
281 def test_with_debug_symbols(self):
282 # A build with debug symbols sets up /CurrentlyBuilding
283 # appropriately, and does not pass DEB_BUILD_OPTIONS.
284@@ -199,7 +208,7 @@
285 'build_debug_symbols': True})
286 os.makedirs(self.chrootdir)
287 self.buildmanager._state = BinaryPackageBuildState.UPDATE
288- self.buildmanager.iterate(0)
289+ yield self.buildmanager.iterate(0)
290 self.assertState(
291 BinaryPackageBuildState.SBUILD,
292 ['sharepath/slavebin/sbuild-package', 'sbuild-package',
293@@ -218,6 +227,7 @@
294 Build-Debug-Symbols: yes
295 """), cb.read())
296
297+ @defer.inlineCallbacks
298 def test_without_debug_symbols(self):
299 # A build with debug symbols sets up /CurrentlyBuilding
300 # appropriately, and passes DEB_BUILD_OPTIONS=noautodbgsym.
301@@ -228,7 +238,7 @@
302 'build_debug_symbols': False})
303 os.makedirs(self.chrootdir)
304 self.buildmanager._state = BinaryPackageBuildState.UPDATE
305- self.buildmanager.iterate(0)
306+ yield self.buildmanager.iterate(0)
307 self.assertState(
308 BinaryPackageBuildState.SBUILD,
309 ['sharepath/slavebin/sbuild-package', 'sbuild-package',
310@@ -248,9 +258,10 @@
311 Purpose: PRIMARY
312 """), cb.read())
313
314+ @defer.inlineCallbacks
315 def test_abort_sbuild(self):
316 # Aborting sbuild kills processes in the chroot.
317- self.startBuild()
318+ yield self.startBuild()
319
320 # Send an abort command. The build manager reaps processes.
321 self.buildmanager.abort()
322@@ -267,10 +278,11 @@
323 self.assertUnmountsSanely()
324 self.assertFalse(self.slave.wasCalled('buildFail'))
325
326+ @defer.inlineCallbacks
327 def test_abort_sbuild_fail(self):
328 # If killing processes in the chroot hangs, the build manager does
329 # its best to clean up and fails the builder.
330- self.startBuild()
331+ yield self.startBuild()
332 sbuild_subprocess = self.buildmanager._subprocess
333
334 # Send an abort command. The build manager reaps processes.
335@@ -300,7 +312,7 @@
336
337 # When sbuild exits, it does not reap processes again, but proceeds
338 # directly to UMOUNT.
339- self.buildmanager.iterate(128 + 9) # SIGKILL
340+ yield self.buildmanager.iterate(128 + 9) # SIGKILL
341 self.assertState(
342 BinaryPackageBuildState.UMOUNT,
343 ['sharepath/slavebin/in-target', 'in-target',
344@@ -308,6 +320,7 @@
345 '--backend=chroot', '--series=warty', '--arch=i386',
346 self.buildid], final=True)
347
348+ @defer.inlineCallbacks
349 def test_abort_between_subprocesses(self):
350 # If a build is aborted between subprocesses, the build manager
351 # pretends that it was terminated by a signal.
352@@ -324,7 +337,7 @@
353 '--backend=chroot', '--series=warty', '--arch=i386',
354 self.buildid], final=False)
355
356- self.buildmanager.iterate(0)
357+ yield self.buildmanager.iterate(0)
358 self.assertState(
359 BinaryPackageBuildState.CLEANUP,
360 ['sharepath/slavebin/in-target', 'in-target',
361@@ -334,10 +347,11 @@
362 final=True)
363 self.assertFalse(self.slave.wasCalled('builderFail'))
364
365+ @defer.inlineCallbacks
366 def test_missing_changes(self):
367 # The build manager recovers if the expected .changes file does not
368 # exist, and considers it a package build failure.
369- self.startBuild()
370+ yield self.startBuild()
371 write_file(
372 os.path.join(self.buildmanager._cachepath, 'buildlog'),
373 "I am a build log.")
374@@ -348,7 +362,7 @@
375 "I am a changes file.")
376
377 # After building the package, reap processes.
378- self.assertScansSanely(SBuildExitCodes.OK)
379+ yield self.assertScansSanely(SBuildExitCodes.OK)
380 self.assertTrue(self.slave.wasCalled('buildFail'))
381 self.assertThat(self.slave, HasWaitingFiles({}))
382
383@@ -564,8 +578,9 @@
384 PkgRelation.parse_relations("foo <!nocheck>, bar <stage1>"),
385 {}))
386
387+ @defer.inlineCallbacks
388 def startDepFail(self, error, dscname=''):
389- self.startBuild(dscname=dscname)
390+ yield self.startBuild(dscname=dscname)
391 write_file(
392 os.path.join(self.buildmanager._cachepath, 'buildlog'),
393 "The following packages have unmet dependencies:\n"
394@@ -574,9 +589,10 @@
395 + ("a" * 4096) + "\n"
396 + "Fail-Stage: install-deps\n")
397
398+ @defer.inlineCallbacks
399 def assertMatchesDepfail(self, error, dep):
400- self.startDepFail(error)
401- self.assertScansSanely(SBuildExitCodes.GIVENBACK)
402+ yield self.startDepFail(error)
403+ yield self.assertScansSanely(SBuildExitCodes.GIVENBACK)
404 self.assertUnmountsSanely()
405 if dep is not None:
406 self.assertFalse(self.slave.wasCalled('buildFail'))
407@@ -585,27 +601,32 @@
408 self.assertFalse(self.slave.wasCalled('depFail'))
409 self.assertTrue(self.slave.wasCalled('buildFail'))
410
411+ @defer.inlineCallbacks
412 def test_detects_depfail(self):
413 # The build manager detects dependency installation failures.
414- self.assertMatchesDepfail(
415+ yield self.assertMatchesDepfail(
416 "enoent but it is not installable", "enoent")
417
418+ @defer.inlineCallbacks
419 def test_detects_versioned_depfail(self):
420 # The build manager detects dependency installation failures.
421- self.assertMatchesDepfail(
422+ yield self.assertMatchesDepfail(
423 "ebadver (< 2.0) but 3.0 is to be installed", "ebadver (< 2.0)")
424
425+ @defer.inlineCallbacks
426 def test_detects_versioned_current_depfail(self):
427 # The build manager detects dependency installation failures.
428- self.assertMatchesDepfail(
429+ yield self.assertMatchesDepfail(
430 "ebadver (< 2.0) but 3.0 is installed", "ebadver (< 2.0)")
431
432+ @defer.inlineCallbacks
433 def test_strips_depfail(self):
434 # The build manager strips qualifications and restrictions from
435 # dependency installation failures.
436- self.assertMatchesDepfail(
437+ yield self.assertMatchesDepfail(
438 "ebadver:any (>= 3.0) but 2.0 is installed", "ebadver (>= 3.0)")
439
440+ @defer.inlineCallbacks
441 def test_uninstallable_deps_analysis_failure(self):
442 # If there are uninstallable build-dependencies and analysis can't
443 # find any missing direct build-dependencies, the build manager
444@@ -618,7 +639,7 @@
445 Version: 1
446 Build-Depends: uninstallable (>= 1)
447 """))
448- self.startDepFail(
449+ yield self.startDepFail(
450 "uninstallable (>= 1) but it is not going to be installed",
451 dscname="123")
452 apt_lists = os.path.join(self.chrootdir, "var", "lib", "apt", "lists")
453@@ -629,11 +650,12 @@
454 Package: uninstallable
455 Version: 1
456 """))
457- self.assertScansSanely(SBuildExitCodes.GIVENBACK)
458+ yield self.assertScansSanely(SBuildExitCodes.GIVENBACK)
459 self.assertUnmountsSanely()
460 self.assertFalse(self.slave.wasCalled('depFail'))
461 self.assertTrue(self.slave.wasCalled('buildFail'))
462
463+ @defer.inlineCallbacks
464 def test_uninstallable_deps_analysis_depfail(self):
465 # If there are uninstallable build-dependencies and analysis reports
466 # some missing direct build-dependencies, the build manager marks
467@@ -645,7 +667,7 @@
468 Version: 1
469 Build-Depends: ebadver (>= 2)
470 """))
471- self.startDepFail(
472+ yield self.startDepFail(
473 "ebadver (>= 2) but it is not going to be installed",
474 dscname="123")
475 apt_lists = os.path.join(self.chrootdir, "var", "lib", "apt", "lists")
476@@ -656,11 +678,12 @@
477 Package: ebadver
478 Version: 1
479 """))
480- self.assertScansSanely(SBuildExitCodes.GIVENBACK)
481+ yield self.assertScansSanely(SBuildExitCodes.GIVENBACK)
482 self.assertUnmountsSanely()
483 self.assertFalse(self.slave.wasCalled('buildFail'))
484 self.assertEqual([(("ebadver (>= 2)",), {})], self.slave.depFail.calls)
485
486+ @defer.inlineCallbacks
487 def test_uninstallable_deps_analysis_mixed_depfail(self):
488 # If there is a mix of definite and dubious dep-wait output, then
489 # the build manager analyses the situation rather than trusting just
490@@ -672,7 +695,7 @@
491 Version: 1
492 Build-Depends: ebadver (>= 2), uninstallable
493 """))
494- self.startBuild(dscname="123")
495+ yield self.startBuild(dscname="123")
496 write_file(
497 os.path.join(self.buildmanager._cachepath, 'buildlog'),
498 "The following packages have unmet dependencies:\n"
499@@ -694,19 +717,20 @@
500 Package: uninstallable
501 Version: 1
502 """))
503- self.assertScansSanely(SBuildExitCodes.GIVENBACK)
504+ yield self.assertScansSanely(SBuildExitCodes.GIVENBACK)
505 self.assertUnmountsSanely()
506 self.assertFalse(self.slave.wasCalled('buildFail'))
507 self.assertEqual([(("ebadver (>= 2)",), {})], self.slave.depFail.calls)
508
509+ @defer.inlineCallbacks
510 def test_depfail_with_unknown_error_converted_to_packagefail(self):
511 # The build manager converts a DEPFAIL to a PACKAGEFAIL if the
512 # missing dependency can't be determined from the log.
513- self.startBuild()
514+ yield self.startBuild()
515 write_file(
516 os.path.join(self.buildmanager._cachepath, 'buildlog'),
517 "E: Everything is broken.\n")
518
519- self.assertScansSanely(SBuildExitCodes.GIVENBACK)
520+ yield self.assertScansSanely(SBuildExitCodes.GIVENBACK)
521 self.assertTrue(self.slave.wasCalled('buildFail'))
522 self.assertFalse(self.slave.wasCalled('depFail'))
523
524=== modified file 'lpbuildd/tests/test_livefs.py'
525--- lpbuildd/tests/test_livefs.py 2017-08-26 09:51:15 +0000
526+++ lpbuildd/tests/test_livefs.py 2018-10-19 06:46:37 +0000
527@@ -1,4 +1,4 @@
528-# Copyright 2013 Canonical Ltd. This software is licensed under the
529+# Copyright 2013-2018 Canonical Ltd. This software is licensed under the
530 # GNU Affero General Public License version 3 (see the file LICENSE).
531
532 __metaclass__ = type
533@@ -10,6 +10,8 @@
534 TempDir,
535 )
536 from testtools import TestCase
537+from testtools.deferredruntest import AsynchronousDeferredRunTest
538+from twisted.internet import defer
539
540 from lpbuildd.livefs import (
541 LiveFilesystemBuildManager,
542@@ -35,6 +37,9 @@
543
544 class TestLiveFilesystemBuildManagerIteration(TestCase):
545 """Run LiveFilesystemBuildManager through its iteration steps."""
546+
547+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
548+
549 def setUp(self):
550 super(TestLiveFilesystemBuildManagerIteration, self).setUp()
551 self.working_dir = self.useFixture(TempDir()).path
552@@ -52,6 +57,7 @@
553 """Retrieve build manager's state."""
554 return self.buildmanager._state
555
556+ @defer.inlineCallbacks
557 def startBuild(self):
558 # The build manager's iterate() kicks off the consecutive states
559 # after INIT.
560@@ -71,7 +77,7 @@
561 self.buildmanager._state = LiveFilesystemBuildState.UPDATE
562
563 # BUILD_LIVEFS: Run the slave's payload to build the live filesystem.
564- self.buildmanager.iterate(0)
565+ yield self.buildmanager.iterate(0)
566 self.assertEqual(
567 LiveFilesystemBuildState.BUILD_LIVEFS, self.getState())
568 expected_command = [
569@@ -85,9 +91,10 @@
570 self.buildmanager.iterate, self.buildmanager.iterators[-1])
571 self.assertFalse(self.slave.wasCalled("chrootFail"))
572
573+ @defer.inlineCallbacks
574 def test_iterate(self):
575 # The build manager iterates a normal build from start to finish.
576- self.startBuild()
577+ yield self.startBuild()
578
579 log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
580 with open(log_path, "w") as log:
581@@ -97,7 +104,7 @@
582 "/build/livecd.ubuntu.manifest", b"I am a manifest file.")
583
584 # After building the package, reap processes.
585- self.buildmanager.iterate(0)
586+ yield self.buildmanager.iterate(0)
587 expected_command = [
588 "sharepath/slavebin/in-target", "in-target",
589 "scan-for-processes",
590@@ -126,9 +133,10 @@
591 self.buildmanager.iterate, self.buildmanager.iterators[-1])
592 self.assertFalse(self.slave.wasCalled("buildFail"))
593
594+ @defer.inlineCallbacks
595 def test_omits_symlinks(self):
596 # Symlinks in the build output are not included in gathered results.
597- self.startBuild()
598+ yield self.startBuild()
599
600 log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
601 with open(log_path, "w") as log:
602@@ -139,7 +147,7 @@
603 self.buildmanager.backend.add_link(
604 "/build/livecd.ubuntu.kernel", "livefs.ubuntu.kernel-generic")
605
606- self.buildmanager.iterate(0)
607+ yield self.buildmanager.iterate(0)
608 self.assertThat(self.slave, HasWaitingFiles.byEquality({
609 "livecd.ubuntu.kernel-generic": b"I am a kernel.",
610 }))
611
612=== modified file 'lpbuildd/tests/test_snap.py'
613--- lpbuildd/tests/test_snap.py 2018-06-05 20:59:48 +0000
614+++ lpbuildd/tests/test_snap.py 2018-10-19 06:46:37 +0000
615@@ -1,4 +1,4 @@
616-# Copyright 2015-2017 Canonical Ltd. This software is licensed under the
617+# Copyright 2015-2018 Canonical Ltd. This software is licensed under the
618 # GNU Affero General Public License version 3 (see the file LICENSE).
619
620 __metaclass__ = type
621@@ -70,6 +70,7 @@
622 """Retrieve build manager's state."""
623 return self.buildmanager._state
624
625+ @defer.inlineCallbacks
626 def startBuild(self, args=None, options=None):
627 # The build manager's iterate() kicks off the consecutive states
628 # after INIT.
629@@ -92,7 +93,7 @@
630 self.buildmanager._state = SnapBuildState.UPDATE
631
632 # BUILD_SNAP: Run the slave's payload to build the snap package.
633- self.buildmanager.iterate(0)
634+ yield self.buildmanager.iterate(0)
635 self.assertEqual(SnapBuildState.BUILD_SNAP, self.getState())
636 expected_command = [
637 "sharepath/slavebin/in-target", "in-target",
638@@ -119,9 +120,10 @@
639 status_file.write('{"revision_id": "dummy"}')
640 self.assertEqual({"revision_id": "dummy"}, self.buildmanager.status())
641
642+ @defer.inlineCallbacks
643 def test_iterate(self):
644 # The build manager iterates a normal build from start to finish.
645- self.startBuild()
646+ yield self.startBuild()
647
648 log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
649 with open(log_path, "w") as log:
650@@ -131,7 +133,7 @@
651 "/build/test-snap/test-snap_0_all.snap", b"I am a snap package.")
652
653 # After building the package, reap processes.
654- self.buildmanager.iterate(0)
655+ yield self.buildmanager.iterate(0)
656 expected_command = [
657 "sharepath/slavebin/in-target", "in-target",
658 "scan-for-processes",
659@@ -159,10 +161,11 @@
660 self.buildmanager.iterate, self.buildmanager.iterators[-1])
661 self.assertFalse(self.slave.wasCalled("buildFail"))
662
663+ @defer.inlineCallbacks
664 def test_iterate_with_manifest(self):
665 # The build manager iterates a build that uploads a manifest from
666 # start to finish.
667- self.startBuild()
668+ yield self.startBuild()
669
670 log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
671 with open(log_path, "w") as log:
672@@ -174,7 +177,7 @@
673 "/build/test-snap/test-snap_0_all.manifest", b"I am a manifest.")
674
675 # After building the package, reap processes.
676- self.buildmanager.iterate(0)
677+ yield self.buildmanager.iterate(0)
678 expected_command = [
679 "sharepath/slavebin/in-target", "in-target",
680 "scan-for-processes",
681@@ -203,10 +206,11 @@
682 self.buildmanager.iterate, self.buildmanager.iterators[-1])
683 self.assertFalse(self.slave.wasCalled("buildFail"))
684
685+ @defer.inlineCallbacks
686 def test_iterate_with_build_source_tarball(self):
687 # The build manager iterates a build that uploads a source tarball
688 # from start to finish.
689- self.startBuild(
690+ yield self.startBuild(
691 {"build_source_tarball": True}, ["--build-source-tarball"])
692
693 log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
694@@ -219,7 +223,7 @@
695 "/build/test-snap.tar.gz", b"I am a source tarball.")
696
697 # After building the package, reap processes.
698- self.buildmanager.iterate(0)
699+ yield self.buildmanager.iterate(0)
700 expected_command = [
701 "sharepath/slavebin/in-target", "in-target",
702 "scan-for-processes",
703
704=== modified file 'lpbuildd/tests/test_sourcepackagerecipe.py'
705--- lpbuildd/tests/test_sourcepackagerecipe.py 2017-08-22 14:53:29 +0000
706+++ lpbuildd/tests/test_sourcepackagerecipe.py 2018-10-19 06:46:37 +0000
707@@ -1,4 +1,4 @@
708-# Copyright 2013-2017 Canonical Ltd. This software is licensed under the
709+# Copyright 2013-2018 Canonical Ltd. This software is licensed under the
710 # GNU Affero General Public License version 3 (see the file LICENSE).
711
712 __metaclass__ = type
713@@ -9,6 +9,8 @@
714 from textwrap import dedent
715
716 from testtools import TestCase
717+from testtools.deferredruntest import AsynchronousDeferredRunTest
718+from twisted.internet import defer
719
720 from lpbuildd.sourcepackagerecipe import (
721 RETCODE_FAILURE_INSTALL_BUILD_DEPS,
722@@ -35,6 +37,9 @@
723
724 class TestSourcePackageRecipeBuildManagerIteration(TestCase):
725 """Run SourcePackageRecipeBuildManager through its iteration steps."""
726+
727+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
728+
729 def setUp(self):
730 super(TestSourcePackageRecipeBuildManagerIteration, self).setUp()
731 self.working_dir = tempfile.mkdtemp()
732@@ -55,6 +60,7 @@
733 """Retrieve build manager's state."""
734 return self.buildmanager._state
735
736+ @defer.inlineCallbacks
737 def startBuild(self, git=False):
738 # The build manager's iterate() kicks off the consecutive states
739 # after INIT.
740@@ -84,7 +90,7 @@
741 self.buildmanager._state = SourcePackageRecipeBuildState.UPDATE
742
743 # BUILD_RECIPE: Run the slave's payload to build the source package.
744- self.buildmanager.iterate(0)
745+ yield self.buildmanager.iterate(0)
746 self.assertEqual(
747 SourcePackageRecipeBuildState.BUILD_RECIPE, self.getState())
748 expected_command = [
749@@ -101,9 +107,10 @@
750 self.buildmanager.iterate, self.buildmanager.iterators[-1])
751 self.assertFalse(self.slave.wasCalled('chrootFail'))
752
753+ @defer.inlineCallbacks
754 def test_iterate(self):
755 # The build manager iterates a normal build from start to finish.
756- self.startBuild()
757+ yield self.startBuild()
758
759 log_path = os.path.join(self.buildmanager._cachepath, 'buildlog')
760 with open(log_path, 'w') as log:
761@@ -121,7 +128,7 @@
762 manifest.write("I am a manifest file.")
763
764 # After building the package, reap processes.
765- self.buildmanager.iterate(0)
766+ yield self.buildmanager.iterate(0)
767 expected_command = [
768 'sharepath/slavebin/in-target', 'in-target',
769 'scan-for-processes',
770@@ -153,9 +160,10 @@
771 self.buildmanager.iterate, self.buildmanager.iterators[-1])
772 self.assertFalse(self.slave.wasCalled('buildFail'))
773
774+ @defer.inlineCallbacks
775 def test_iterate_BUILD_RECIPE_install_build_deps_depfail(self):
776 # The build manager can detect dependency wait states.
777- self.startBuild()
778+ yield self.startBuild()
779
780 log_path = os.path.join(self.buildmanager._cachepath, 'buildlog')
781 with open(log_path, 'w') as log:
782@@ -166,7 +174,7 @@
783 " but it is not going to be installed.\n")
784
785 # The buildmanager calls depFail correctly and reaps processes.
786- self.buildmanager.iterate(RETCODE_FAILURE_INSTALL_BUILD_DEPS)
787+ yield self.buildmanager.iterate(RETCODE_FAILURE_INSTALL_BUILD_DEPS)
788 expected_command = [
789 'sharepath/slavebin/in-target', 'in-target',
790 'scan-for-processes',
791@@ -196,17 +204,18 @@
792 self.buildmanager.iterate, self.buildmanager.iterators[-1])
793 self.assertFalse(self.slave.wasCalled('buildFail'))
794
795+ @defer.inlineCallbacks
796 def test_iterate_BUILD_RECIPE_install_build_deps_buildfail(self):
797 # If the build manager cannot detect a dependency wait from a
798 # build-dependency installation failure, it fails the build.
799- self.startBuild()
800+ yield self.startBuild()
801
802 log_path = os.path.join(self.buildmanager._cachepath, 'buildlog')
803 with open(log_path, 'w') as log:
804 log.write("I am a failing build log.")
805
806 # The buildmanager calls buildFail correctly and reaps processes.
807- self.buildmanager.iterate(RETCODE_FAILURE_INSTALL_BUILD_DEPS)
808+ yield self.buildmanager.iterate(RETCODE_FAILURE_INSTALL_BUILD_DEPS)
809 expected_command = [
810 'sharepath/slavebin/in-target', 'in-target',
811 'scan-for-processes',
812@@ -234,8 +243,9 @@
813 self.assertEqual(
814 self.buildmanager.iterate, self.buildmanager.iterators[-1])
815
816+ @defer.inlineCallbacks
817 def test_iterate_git(self):
818 # Starting a git-based recipe build passes the correct option. (The
819 # rest of the build is identical to bzr-based recipe builds from the
820 # build manager's point of view.)
821- self.startBuild(git=True)
822+ yield self.startBuild(git=True)
823
824=== modified file 'lpbuildd/tests/test_translationtemplatesbuildmanager.py'
825--- lpbuildd/tests/test_translationtemplatesbuildmanager.py 2017-09-08 15:57:18 +0000
826+++ lpbuildd/tests/test_translationtemplatesbuildmanager.py 2018-10-19 06:46:37 +0000
827@@ -1,4 +1,4 @@
828-# Copyright 2010-2017 Canonical Ltd. This software is licensed under the
829+# Copyright 2010-2018 Canonical Ltd. This software is licensed under the
830 # GNU Affero General Public License version 3 (see the file LICENSE).
831
832 __metaclass__ = type
833@@ -10,6 +10,8 @@
834 TempDir,
835 )
836 from testtools import TestCase
837+from testtools.deferredruntest import AsynchronousDeferredRunTest
838+from twisted.internet import defer
839
840 from lpbuildd.target.generate_translation_templates import (
841 RETCODE_FAILURE_BUILD,
842@@ -39,6 +41,9 @@
843
844 class TestTranslationTemplatesBuildManagerIteration(TestCase):
845 """Run TranslationTemplatesBuildManager through its iteration steps."""
846+
847+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
848+
849 def setUp(self):
850 super(TestTranslationTemplatesBuildManagerIteration, self).setUp()
851 self.working_dir = self.useFixture(TempDir()).path
852@@ -57,6 +62,7 @@
853 """Retrieve build manager's state."""
854 return self.buildmanager._state
855
856+ @defer.inlineCallbacks
857 def test_iterate(self):
858 # Two iteration steps are specific to this build manager.
859 url = 'lp:~my/branch'
860@@ -74,7 +80,7 @@
861
862 # GENERATE: Run the slave's payload, the script that generates
863 # templates.
864- self.buildmanager.iterate(0)
865+ yield self.buildmanager.iterate(0)
866 self.assertEqual(
867 TranslationTemplatesBuildState.GENERATE, self.getState())
868 expected_command = [
869@@ -95,7 +101,7 @@
870 outfile_path, b"I am a template tarball. Seriously.")
871
872 # After generating templates, reap processes.
873- self.buildmanager.iterate(0)
874+ yield self.buildmanager.iterate(0)
875 expected_command = [
876 'sharepath/slavebin/in-target', 'in-target',
877 'scan-for-processes',
878@@ -128,6 +134,7 @@
879 self.buildmanager.iterate, self.buildmanager.iterators[-1])
880 self.assertFalse(self.slave.wasCalled('buildFail'))
881
882+ @defer.inlineCallbacks
883 def test_iterate_fail_GENERATE_install(self):
884 # See that a GENERATE that fails at the install step is handled
885 # properly.
886@@ -141,7 +148,7 @@
887 self.buildmanager._state = TranslationTemplatesBuildState.GENERATE
888
889 # The buildmanager fails and reaps processes.
890- self.buildmanager.iterate(RETCODE_FAILURE_INSTALL)
891+ yield self.buildmanager.iterate(RETCODE_FAILURE_INSTALL)
892 self.assertEqual(
893 TranslationTemplatesBuildState.GENERATE, self.getState())
894 expected_command = [
895@@ -169,6 +176,7 @@
896 self.assertEqual(
897 self.buildmanager.iterate, self.buildmanager.iterators[-1])
898
899+ @defer.inlineCallbacks
900 def test_iterate_fail_GENERATE_build(self):
901 # See that a GENERATE that fails at the build step is handled
902 # properly.
903@@ -182,7 +190,7 @@
904 self.buildmanager._state = TranslationTemplatesBuildState.GENERATE
905
906 # The buildmanager fails and reaps processes.
907- self.buildmanager.iterate(RETCODE_FAILURE_BUILD)
908+ yield self.buildmanager.iterate(RETCODE_FAILURE_BUILD)
909 expected_command = [
910 'sharepath/slavebin/in-target', 'in-target',
911 'scan-for-processes',
912
913=== modified file 'lpbuildd/translationtemplates.py'
914--- lpbuildd/translationtemplates.py 2017-09-08 15:57:18 +0000
915+++ lpbuildd/translationtemplates.py 2018-10-19 06:46:37 +0000
916@@ -1,4 +1,4 @@
917-# Copyright 2010-2017 Canonical Ltd. This software is licensed under the
918+# Copyright 2010-2018 Canonical Ltd. This software is licensed under the
919 # GNU Affero General Public License version 3 (see the file LICENSE).
920
921 __metaclass__ = type
922@@ -63,7 +63,7 @@
923 """Template generation finished."""
924 if retcode == 0:
925 # It worked! Now let's bring in the harvest.
926- self.gatherResults()
927+ return self.deferGatherResults()
928 else:
929 if not self.alreadyfailed:
930 if retcode == RETCODE_FAILURE_INSTALL:

Subscribers

People subscribed via source and target branches

to all changes: